{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 无隐层神经网络："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 引包\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**注：x为向量**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tanh激活函数：\n",
    "$$tanh(x) = \\frac{e^x - e^{-x}}{e^x + e^{-x}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### softmax激活函数：\n",
    "$$softmax(x) = \\frac{e^x}{\\sum^{k}_{0}{e^{x_{[k]}}}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 激活函数定义\n",
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "def softmax(x):\n",
    "    exp = np.exp(x - x.max())\n",
    "    return exp / exp.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义卷积信息，第一层是784个神经元，第二层10个神经元，为了示范结构简单，所以没有隐藏层\n",
    "dimensions = [28 * 28, 10]\n",
    "# 定义第一层的激活函数是tanh，第二层的激活函数是softmax\n",
    "activation = [tanh, softmax]\n",
    "# 定义每层初始化的b、w值范围，示范结构比较简单，第一层没有w\n",
    "distribution = [\n",
    "    {'b': [0, 0]},\n",
    "    {'b': [0, 0], 'w': [-math.sqrt(6 / (dimensions[0] + dimensions[1])),\n",
    "                        math.sqrt(6 / (dimensions[0] + dimensions[1]))]}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用随机数初始化layer层的b参数\n",
    "def init_parameters_b(layer):\n",
    "    dist = distribution[layer]['b']\n",
    "    return np.random.rand(dimensions[layer]) * (dist[1] - dist[0]) + dist[0]\n",
    "# 用随机数初始化layer层的w参数\n",
    "def init_parameters_w(layer):\n",
    "    dist = distribution[layer]['w']\n",
    "    return np.random.rand(dimensions[layer - 1], dimensions[layer]) * (dist[1] - dist[0]) + dist[0]\n",
    "# 总的初始化函数，调用之后初始化所有层\n",
    "def init_parameters():\n",
    "    parameter = []\n",
    "    # 循环遍历每层并初始化\n",
    "    for i in range(len(distribution)):\n",
    "        layer_parameter = {}\n",
    "        for j in distribution[i].keys():\n",
    "            if j == 'b':\n",
    "                layer_parameter['b'] = init_parameters_b(i)\n",
    "                continue\n",
    "            if j == 'w':\n",
    "                layer_parameter['w'] = init_parameters_w(i)\n",
    "                continue\n",
    "        parameter.append(layer_parameter)\n",
    "    return parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化步骤\n",
    "parameters = init_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 先写了个预测函数\n",
    "def predict(img, parameters):\n",
    "    # 第1层（此处口头上说的层数是变量层数+1）的输入是图像像素（扁平化的28*28）加上第1层的b\n",
    "    l0_in = img + parameters[0]['b']\n",
    "    # 第一层的输出是第一层的输入作为参数传给第一层的激活函数之后的结果\n",
    "    l0_out = activation[0](l0_in)\n",
    "    # 第二层（输出层）的输入等于第一层的输出点乘第二层的w加上第二层的b\n",
    "    l1_in = np.dot(l0_out, parameters[1]['w']) + parameters[1]['b']\n",
    "    # 第二层的输出等于第二层的输入传给第二层的激活函数\n",
    "    l1_out = activation[1](l1_in)\n",
    "    return l1_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 引入pathlib库做路径处理\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 各类数据路径\n",
    "dataset_path = Path(\"./MNIST\")\n",
    "train_img_path = dataset_path / 'train-images.idx3-ubyte'\n",
    "train_lab_path = dataset_path / 'train-labels.idx1-ubyte'\n",
    "test_img_path = dataset_path / 't10k-images.idx3-ubyte'\n",
    "test_lab_path = dataset_path / 't10k-labels.idx1-ubyte'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 划分训练集、测试集、验证集大小\n",
    "train_num = 50000\n",
    "valid_num = 10000\n",
    "test_num = 10000\n",
    "\n",
    "# import struct\n",
    "# 各种读入\n",
    "with open(train_img_path, 'rb') as trainf:\n",
    "    # print(struct.unpack('>4i', trainf.read(16)))\n",
    "    trainf.read(16)\n",
    "    tmp_img = np.fromfile(trainf, dtype=np.uint8).reshape(-1, 28 * 28) / 255\n",
    "    train_img = tmp_img[:train_num]\n",
    "    valid_img = tmp_img[train_num:]\n",
    "\n",
    "with open(test_img_path, 'rb') as testf:\n",
    "    # print(struct.unpack('>4i', trainf.read(16)))\n",
    "    testf.read(16)\n",
    "    test_img = np.fromfile(testf, dtype=np.uint8).reshape(-1, 28 * 28) / 255\n",
    "with open(train_lab_path, 'rb') as trainlabf:\n",
    "    trainlabf.read(8)\n",
    "    tmp_lab = np.fromfile(trainlabf, dtype=np.uint8)\n",
    "    train_lab = tmp_lab[:train_num]\n",
    "    valid_lab = tmp_lab[train_num:]\n",
    "with open(test_lab_path, 'rb') as testlabf:\n",
    "    testlabf.read(8)\n",
    "    test_lab = np.fromfile(testlabf, dtype=np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "# 显示标签和图片\n",
    "def show_train(index):\n",
    "    plt.imshow(train_img[index].reshape(28, 28), cmap='gray')\n",
    "    print(f'label: {train_lab[index]}')\n",
    "def show_test(index):\n",
    "    plt.imshow(test_img[index].reshape(28, 28), cmap='gray')\n",
    "    print(f'label: {test_lab[index]}')\n",
    "def show_valid(index):\n",
    "    plt.imshow(valid_img[index].reshape(28, 28), cmap='gray')\n",
    "    print(f'label: {valid_lab[index]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label: 7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANiklEQVR4nO3df6xU9ZnH8c9nlRKkRGGJLmvZ7Q+JWTWRbgjZ4EZZSxt/JVBiN+WPSg32NloNxsaV+CMl8R907TYbY5pcI5SurOSaFku0WYuI0fpHFQ0i9lp0CVspV9guJJWEpKs++8c9bG7xzncu8+vM5Xm/kpuZOc+cc56Mfjhn5jtzvo4IATj9/VndDQDoDcIOJEHYgSQIO5AEYQeSOLOXO7PNR/9Al0WEx1ve1pHd9lW2f2P7Xdtr2tkWgO5yq+Psts+QtFfSlyUdkPSqpBUR8evCOhzZgS7rxpF9oaR3I2JfRPxR0mZJS9vYHoAuaifs50t6b8zjA9WyP2F7wPZO2zvb2BeANrXzAd14pwqfOE2PiEFJgxKn8UCd2jmyH5A0d8zjz0g62F47ALqlnbC/Kmme7c/Z/pSkr0va2pm2AHRay6fxEfGh7VslPSvpDEnrI+KtjnUGoKNaHnpraWe8Zwe6ritfqgEweRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmh5fnZJsr1f0geSPpL0YUQs6ERTADqvrbBX/iEift+B7QDoIk7jgSTaDXtI+oXt12wPjPcE2wO2d9re2ea+ALTBEdH6yvZfRsRB2+dK2ibptoh4sfD81ncGYEIiwuMtb+vIHhEHq9vDkrZIWtjO9gB0T8thtz3d9owT9yV9RdKeTjUGoLPa+TT+PElbbJ/Yzr9HxH90pCsAHdfWe/ZT3hnv2YGu68p7dgCTB2EHkiDsQBKEHUiCsANJdOKHMMCkc8MNNxTra9euLdbvv//+Yn3Dhg2n2lLXcWQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZz/NrVu3rli/4IILivUHH3ywWH/llVdOuadeuemmmxrWHnnkkeK6U6ZM6XQ7tePIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM4+CZx5Zvk/09atWxvWlixZ0ta2jx8/XqwPDQ0V68PDw8V6ybJly4r1e+65p1ifMWNGy/t+/PHHi/VNmza1vO26cGQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSSYxXUSWLlyZbHej9co73e7d+8u1pcvX16s79u3r5PtdFTLs7jaXm/7sO09Y5bNsr3N9jvV7cxONgug8yZyGv8jSVedtGyNpO0RMU/S9uoxgD7WNOwR8aKkIyctXippY3V/o6Ty9xoB1K7V78afFxEjkhQRI7bPbfRE2wOSBlrcD4AO6foPYSJiUNKgxAd0QJ1aHXo7ZHuOJFW3hzvXEoBuaDXsWyWdGA9aKelnnWkHQLc0HWe3/YSkxZJmSzok6XuSnpI0JOmvJP1W0tci4uQP8cbbFqfx47j44ouL9WbXZp82bVrD2ksvvVRc97nnnivWL7nkkmJ90aJFxXrJOeecU6xPnz695W1L0ttvv92wdssttxTXfeGFF9rad50ajbM3fc8eESsalL7UVkcAeoqvywJJEHYgCcIOJEHYgSQIO5AEl5LugbPPPrtYf+ihh4r10tCaJNnjjrRIkh599NHius0umdxNl19+ebG+ZcuWYr3ZtMp33nlnw9pkHlprFUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYeuPLKK4v1xYsXF+vNfob8zDPPtFTrhalTpzasXX311cV1zzrrrGL96aefLtaff/75Yj0bjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7D0wa9asYr00Fi1Jw8PDxfptt93WsHb06NHiuu2aMWNGsX7vvfc2rJV+by5Jb7zxRrF+4403FuvHjx8v1rPhyA4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDO3gPNpk1++OGHi/X9+/e3Ve+mK664olhvNhZeMjg4WKwfO3as5W1n1PTIbnu97cO294xZttb272zvqv6u6W6bANo1kdP4H0m6apzlP4iI+dXfzzvbFoBOaxr2iHhR0pEe9AKgi9r5gO5W27ur0/yZjZ5ke8D2Tts729gXgDa1GvYfSvqCpPmSRiR9v9ETI2IwIhZExIIW9wWgA1oKe0QcioiPIuJjSY9KWtjZtgB0Wkthtz1nzMOvStrT6LkA+oObXZPc9hOSFkuaLemQpO9Vj+dLCkn7JX07Ikaa7swu7wx959JLLy3Wh4aGivV58+Y1rN13333FdR977LFi/f333y/Ws4oIj7e86ZdqImLFOIvL/xUA9B2+LgskQdiBJAg7kARhB5Ig7EAS/MQVRcuXLy/WS0NrzWzYsKFYZ2itsziyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLOf5mbPnl2sr1+/vli/7rrrivVm0ypfe+21DWsjI01/FY0O4sgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzn6aW7duXbHebBx97969xfodd9xRrB88eLBYR+9wZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJBhnPw1ceOGFDWsXXXRRW9t+8skni/UdO3a0tX30TtMju+25tnfYHrb9lu3V1fJZtrfZfqe6ndn9dgG0aiKn8R9K+m5E/I2kv5P0HdsXSVojaXtEzJO0vXoMoE81DXtEjETE69X9DyQNSzpf0lJJG6unbZS0rFtNAmjfKb1nt/1ZSV+U9CtJ50XEiDT6D4LtcxusMyBpoL02AbRrwmG3/WlJP5F0e0T8wfaE1ouIQUmD1TailSYBtG9CQ2+2p2g06Jsi4qfV4kO251T1OZIOd6dFAJ3giPLB1qOH8I2SjkTE7WOW/7Ok/4mIdbbXSJoVEf/UZFsc2Vswffr0Yn3z5s0Na6VLOUvSnj17ivVFixYV68eOHSvW0XsRMe5p90RO4y+T9A1Jb9reVS27W9I6SUO2V0n6raSvdaJRAN3RNOwR8UtJjd6gf6mz7QDoFr4uCyRB2IEkCDuQBGEHkiDsQBL8xLUPTJ06tVjfsGFDsb5kyZKGtWbfo3jggQeKdcbRTx8c2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZ+8DNN99crF9//fUtb/vZZ58t1jdt2tTytjG5cGQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSSaXje+oztLet34ZtduHxoaKtanTZtWrD/11FMNa6tWrSque/To0WIdk0+j68ZzZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJJr+nt32XEk/lvQXkj6WNBgR/2p7raRvSfrv6ql3R8TPu9XoZDZlypRivdk4+ssvv1ysr169umGNcXScMJGLV3wo6bsR8brtGZJes72tqv0gIh7qXnsAOmUi87OPSBqp7n9ge1jS+d1uDEBnndJ7dtuflfRFSb+qFt1qe7ft9bZnNlhnwPZO2zvb6hRAWyYcdtuflvQTSbdHxB8k/VDSFyTN1+iR//vjrRcRgxGxICIWdKBfAC2aUNhtT9Fo0DdFxE8lKSIORcRHEfGxpEclLexemwDa1TTsti3pMUnDEfEvY5bPGfO0r0ra0/n2AHTKRD6Nv0zSNyS9aXtXtexuSStsz5cUkvZL+nZXOoTuuuuuYv29997rUSeYzCbyafwvJY33+1jG1IFJhG/QAUkQdiAJwg4kQdiBJAg7kARhB5LgUtLAaYZLSQPJEXYgCcIOJEHYgSQIO5AEYQeSIOxAEhP5PXsn/V7Sf415PLta1o/6tbd+7Uuit1Z1sre/blTo6ZdqPrFze2e/XpuuX3vr174kemtVr3rjNB5IgrADSdQd9sGa91/Sr731a18SvbWqJ73V+p4dQO/UfWQH0COEHUiilrDbvsr2b2y/a3tNHT00Ynu/7Tdt76p7frpqDr3DtveMWTbL9jbb71S3486xV1Nva23/rnrtdtm+pqbe5treYXvY9lu2V1fLa33tCn315HXr+Xt222dI2ivpy5IOSHpV0oqI+HVPG2nA9n5JCyKi9i9g2L5c0jFJP46IS6plD0o6EhHrqn8oZ0ZEeRaJ3vW2VtKxuqfxrmYrmjN2mnFJyyR9UzW+doW+/lE9eN3qOLIvlPRuROyLiD9K2ixpaQ199L2IeFHSkZMWL5W0sbq/UaP/s/Rcg976QkSMRMTr1f0PJJ2YZrzW167QV0/UEfbzJY2dr+iA+mu+95D0C9uv2R6ou5lxnBcRI9Lo/zySzq25n5M1nca7l06aZrxvXrtWpj9vVx1hH+/6WP00/ndZRPytpKslfac6XcXETGga714ZZ5rxvtDq9OftqiPsByTNHfP4M5IO1tDHuCLiYHV7WNIW9d9U1IdOzKBb3R6uuZ//10/TeI83zbj64LWrc/rzOsL+qqR5tj9n+1OSvi5paw19fILt6dUHJ7I9XdJX1H9TUW+VtLK6v1LSz2rs5U/0yzTejaYZV82vXe3Tn0dEz/8kXaPRT+T/U9I9dfTQoK/PS3qj+nur7t4kPaHR07r/1egZ0SpJfy5pu6R3qttZfdTbv0l6U9JujQZrTk29/b1G3xrulrSr+rum7teu0FdPXje+LgskwTfogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJ/wMeyCjCrxNZtAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_train(np.random.randint(train_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label: 4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANrklEQVR4nO3df6gd9ZnH8c9nY6+GtGqykhhtMLXkj60/NlmjLFqDi7QY/4kRuiRojSh7q1RoYMVVF6ywKGHddt2/CrcoTaRrLRo1hNpGQ11XhCZXycbYmPqD2KS5JEo0TQRpfjz7x53INd4z53pm5pyT+7xfcDnnzHNm5mH0k5lz5sx8HRECMPn9Va8bANAdhB1IgrADSRB2IAnCDiRxSjdXZpuv/oGGRYTHm15pz277Gts7bL9t++4qywLQLHd6nt32FEl/kPQtSbslbZa0PCJ+XzIPe3agYU3s2S+T9HZEvBsRf5H0C0lLKiwPQIOqhP1cSbvGvN5dTPsM24O2h20PV1gXgIqqfEE33qHC5w7TI2JI0pDEYTzQS1X27LslzRnz+quS9lRrB0BTqoR9s6R5tr9me0DSMknr6mkLQN06PoyPiCO275D0G0lTJD0aEW/U1hmAWnV86q2jlfGZHWhcIz+qAXDyIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJjodsBpp2xhlnlNYffPDB0vrmzZtb1jZs2FA67549e0rrJ6NKYbe9U9JBSUclHYmIhXU0BaB+dezZ/yEiPqhhOQAaxGd2IImqYQ9JG2y/antwvDfYHrQ9bHu44roAVFD1MP6KiNhje6ak522/GREvjX1DRAxJGpIk21FxfQA6VGnPHhF7isd9kp6WdFkdTQGoX8dhtz3N9leOP5f0bUnb6moMQL2qHMbPkvS07ePL+e+I+HUtXQGSBgfH/RroU5dccklpfcaMGS1rAwMDpfMODQ2V1k9GHYc9It6V9Lc19gKgQZx6A5Ig7EAShB1IgrADSRB2IAlHdO9HbfyCrjOLFy8urT/55JMta4cPHy6d98wzz+yopzqUnRqTpOHh8l9YHzhwoLR+6qmntqxdfvnlpfN+9NFHpfV+FhEebzp7diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgltJnwSWLl1aWp86dWrL2imn9O9/4htuuKG0Pnfu3ErLX7lyZcvayXwevVPs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgif49CZvIPffcU1q/5ZZbOl72pk2bOp63afPmzas0/7Fjx0rr77zzTqXlTzbs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCe4b3wXz588vrT/33HOl9bPPPru0vn///pa1JUuWlM778ssvl9arWrRoUcvaCy+8UDpvu2vxH3vssdL6ihUrSuuTVcf3jbf9qO19treNmTbD9vO23yoep9fZLID6TeQw/meSrjlh2t2SNkbEPEkbi9cA+ljbsEfES5JOPE5cIml18Xy1pOtq7gtAzTr9bfysiBiRpIgYsT2z1RttD0oa7HA9AGrS+IUwETEkaUjK+wUd0A86PfW21/ZsSSoe99XXEoAmdBr2dZKOn9dYIenZetoB0JS2h/G2H5d0laSzbO+W9ENJqyT90vatkv4o6TtNNtnvBgYGSusPPPBAab3defR2v4VYs2ZNy1rT59HbOf/881vWqt7Tvt12xWe13doRsbxF6eqaewHQIH4uCyRB2IEkCDuQBGEHkiDsQBLcSroGt912W2l98eLFpfV2p9ba3RL54YcfLq03adq0aaX1smGTqzp06FBjy56M2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKcZ5+g8847r2Wt6qWW9rh3/v3UfffdV1p/7733Kq2/irvuuqu0fvHFF3e87I0bN5bW33///Y6XnRF7diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgvPsE3TjjTe2rLW7pruddteztxt2+eqrW9/o94knniid98MPPyytt3P99ddXmr9MuyGZDx8+XFqfObPlqGTaty/fuCbs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCbc7x1vryuzuraxmH3/8ccva1KlTu9hJHgcOHCitr1q1qrR+++23t6xdeeWVpfPu2rWrtN7PImLcGyS03bPbftT2Ptvbxky73/afbG8p/q6ts1kA9ZvIYfzPJF0zzvT/jIj5xd+v6m0LQN3ahj0iXpK0vwu9AGhQlS/o7rC9tTjMn97qTbYHbQ/bHq6wLgAVdRr2n0j6uqT5kkYk/ajVGyNiKCIWRsTCDtcFoAYdhT0i9kbE0Yg4Jumnki6rty0Adeso7LZnj3m5VNK2Vu8F0B/aXs9u+3FJV0k6y/ZuST+UdJXt+ZJC0k5J32uwx76wbNmylrV253s3bdpUWi+77loqP8cvSRdccEHL2umnn14679atW0vr7caWr2LHjh2l9WPHjpXWb7rpptL62rVrW9YOHjxYOu9k1DbsEbF8nMmPNNALgAbxc1kgCcIOJEHYgSQIO5AEYQeS4BLXSe6cc86pVG932rCdsmGX253WO3LkSKV1Z9XxJa4AJgfCDiRB2IEkCDuQBGEHkiDsQBKEHUiCIZsnuZGRkdL6ihUrGl3/+vXrW9Y4j95d7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAmuZ5/kBgYGSuuffPJJpeW3G9p4wYIFLWv79zOEYBO4nh1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkuB69knutNNOa3T5L774Ymmdc+n9o+2e3fYc27+1vd32G7Z/UEyfYft5228Vj9ObbxdApyZyGH9E0j9HxN9I+ntJ37f9DUl3S9oYEfMkbSxeA+hTbcMeESMR8Vrx/KCk7ZLOlbRE0uribaslXddUkwCq+0Kf2W3PlbRA0u8kzYqIEWn0HwTbM1vMMyhpsFqbAKqacNhtf1nSU5JWRsSf7XF/a/85ETEkaahYBhfCAD0yoVNvtr+k0aD/PCLWFpP32p5d1GdL2tdMiwDq0HbP7tFd+COStkfEj8eU1klaIWlV8fhsIx2ikptvvrnR5T/zzDONLh/1mchh/BWSvivpddtbimn3ajTkv7R9q6Q/SvpOMy0CqEPbsEfEy5JafUC/ut52ADSFn8sCSRB2IAnCDiRB2IEkCDuQBLeSngSmTJnSsvbKK6+UznvppZeW1t98883S+kUXXVRaP3r0aGkd9eNW0kByhB1IgrADSRB2IAnCDiRB2IEkCDuQBLeSngTKznVfeOGFlZY9Y8aM0vpE71iE3mPPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJcJ59EtiyZUvL2kMPPVQ676JFi0rrd955Z2n9yJEjpXX0D/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BE2/vG254jaY2ksyUdkzQUEf9l+35J/yTp/eKt90bEr9osi/vGAw1rdd/4iYR9tqTZEfGa7a9IelXSdZL+UdKhiPiPiTZB2IHmtQr7RMZnH5E0Ujw/aHu7pHPrbQ9A077QZ3bbcyUtkPS7YtIdtrfaftT29BbzDNoetj1cqVMAlUx4rDfbX5b0P5IeiIi1tmdJ+kBSSPo3jR7q39JmGRzGAw3r+DO7JNn+kqT1kn4TET8epz5X0vqIKL27IWEHmtfxwI4evX3oI5K2jw168cXdcUslbavaJIDmTOTb+G9K+l9Jr2v01Jsk3StpuaT5Gj2M3ynpe8WXeWXLYs8ONKzSYXxdCDvQPMZnB5Ij7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJNHtIZs/kPTemNdnFdP6Ub/21q99SfTWqTp7O69VoavXs39u5fZwRCzsWQMl+rW3fu1LordOdas3DuOBJAg7kESvwz7U4/WX6dfe+rUvid461ZXeevqZHUD39HrPDqBLCDuQRE/Cbvsa2ztsv2377l700IrtnbZft72l1+PTFWPo7bO9bcy0Gbaft/1W8TjuGHs96u1+238qtt0W29f2qLc5tn9re7vtN2z/oJje021X0ldXtlvXP7PbniLpD5K+JWm3pM2SlkfE77vaSAu2d0paGBE9/wGG7UWSDklac3xoLdv/Lml/RKwq/qGcHhH/0ie93a8vOIx3Q721Gmb8ZvVw29U5/HknerFnv0zS2xHxbkT8RdIvJC3pQR99LyJekrT/hMlLJK0unq/W6P8sXdeit74QESMR8Vrx/KCk48OM93TblfTVFb0I+7mSdo15vVv9Nd57SNpg+1Xbg71uZhyzjg+zVTzO7HE/J2o7jHc3nTDMeN9su06GP6+qF2Efb2iafjr/d0VE/J2kxZK+XxyuYmJ+IunrGh0DcETSj3rZTDHM+FOSVkbEn3vZy1jj9NWV7daLsO+WNGfM669K2tODPsYVEXuKx32Sntbox45+svf4CLrF474e9/OpiNgbEUcj4pikn6qH264YZvwpST+PiLXF5J5vu/H66tZ260XYN0uaZ/trtgckLZO0rgd9fI7tacUXJ7I9TdK31X9DUa+TtKJ4vkLSsz3s5TP6ZRjvVsOMq8fbrufDn0dE1/8kXavRb+TfkfSvveihRV/nS/q/4u+NXvcm6XGNHtYd1ugR0a2S/lrSRklvFY8z+qi3xzQ6tPdWjQZrdo96+6ZGPxpulbSl+Lu219uupK+ubDd+LgskwS/ogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJ/weIbzqrgEWn1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_valid(np.random.randint(valid_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label: 5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANRElEQVR4nO3df6gd9ZnH8c/HbCOaBE38tdGGTSsia1Y2WRJdSVlcSkNWAlqh2vyxWCve/lGlgZXd4ApRpCCrXdk/tHJDtenStRRiqZSEKqFsXITgNSSaH5voSjZNE5LNRmwigSbx2T/uZLnGe77nembOmZM87xdczjnznJl5GPLJzJw5c76OCAG48F3UdgMABoOwA0kQdiAJwg4kQdiBJP5okCuzzUf/QJ9FhCebXmvPbnu57T2237e9us6yAPSXe73ObnuapL2SvibpgKS3JK2MiF2FedizA33Wjz37LZLej4gPIuIPkn4m6c4aywPQR3XCfp2k3054faCa9im2R2yP2R6rsS4ANdX5gG6yQ4XPHKZHxKikUYnDeKBNdfbsByTNm/D6i5IO1msHQL/UCftbkm6w/SXb0yV9U9KrzbQFoGk9H8ZHxGnbD0n6taRpkl6MiJ2NdQagUT1feutpZZyzA33Xly/VADh/EHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BEz0M248Iwffr0Yv3ZZ58t1m+77bZi/cknn+xY2759e3Hebi655JJi/cEHH+xYu/fee4vzXnHFFcX6888/X6zv3FkevXzt2rXFej/UCrvtfZKOSzoj6XRELG6iKQDNa2LP/tcRcbSB5QDoI87ZgSTqhj0kvWb7bdsjk73B9ojtMdtjNdcFoIa6h/FLI+Kg7aslvW77PyNi88Q3RMSopFFJsh011wegR7X27BFxsHo8IukXkm5poikAzes57LZn2J519rmkZZJ2NNUYgGY5orcja9tf1vjeXBo/Hfi3iPh+l3k4jB8y8+fPL9bHxsoftcyePbtYt92x1uu/vfPBm2++WawvW7asY+3kyZO11h0Rk270ns/ZI+IDSX/ec0cABopLb0AShB1IgrADSRB2IAnCDiTBLa7JLVq0qFjvdgtsVh9++GGxvmHDhmL9zJkzTbYzJezZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJrrNf4Eq3mErS3XffXazPmDGj1vpPnDjRsfbxxx8X5+32c8ybNm3qqSdJ2rt3b7G+cePGYr3b7bl1b1PtB/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BEzz8l3dPK+CnpgZs5c2ax/tFHH9Va/qpVq4r11157rWNtz549tdaNyXX6KWn27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBPezo5Zu94VzLX14dN2z237R9hHbOyZMm2P7ddvvVY/lQboBtG4qh/E/lrT8nGmrJW2KiBskbapeAxhiXcMeEZslHTtn8p2S1lXP10m6q+G+ADSs13P2ayLikCRFxCHbV3d6o+0RSSM9rgdAQ/r+AV1EjEoalbgRBmhTr5feDtueK0nV45HmWgLQD72G/VVJ91XP75P0y2baAdAvXQ/jbb8s6XZJV9o+IGmNpKck/dz2A5L2S/pGP5tE75YsWdLX5c+ZM6evy0dzuoY9IlZ2KH214V4A9BFflwWSIOxAEoQdSIKwA0kQdiAJbnG9wK1YsaLW/MePHy/WN2/eXGv5GBz27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBNfZL3ALFiyoNf/+/fuL9RMnTtRaPgaHPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4k4YgY3Mrswa0MkqSlS5cW6xs3bizWZ82aVawvXLiwWN++fXuxjuZFhCeb3nXPbvtF20ds75gw7XHbv7O9rfq7o8lmATRvKofxP5a0fJLpz0bEwupvQ7NtAWha17BHxGZJxwbQC4A+qvMB3UO236kO82d3epPtEdtjtsdqrAtATb2G/YeSrpe0UNIhST/o9MaIGI2IxRGxuMd1AWhAT2GPiMMRcSYiPpG0VtItzbYFoGk9hd323Akvvy5pR6f3AhgOXX833vbLkm6XdKXtA5LWSLrd9kJJIWmfpO/0sUfUcOrUqWJ9x47y/9O33nprsc519vNH17BHxMpJJv+oD70A6CO+LgskQdiBJAg7kARhB5Ig7EASDNl8get2ae3GG2+stfydO3fWmh+Dw54dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5LgOvsU3XTTTR1rW7ZsKc7b7Vr0E088Uax3+7nnkosuKv9/fvnll/e8bEk6evRorfkxOOzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJrrNP0cjISMfapZdeWpx3yZIlxfr69euL9aeffrpYX7NmTbEOSOzZgTQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJrrNP0datWzvWDh48WJz32muvLdYvvvjiYv2qq64q1ktOnjxZrD/zzDPF+iOPPFKsd/vd+X379hXrGJyue3bb82z/xvZu2zttf6+aPsf267bfqx5n979dAL2aymH8aUl/FxF/KukvJX3X9k2SVkvaFBE3SNpUvQYwpLqGPSIORcTW6vlxSbslXSfpTknrqretk3RXv5oEUN/nOme3PV/SIklbJF0TEYek8f8QbF/dYZ4RSZ2/WA5gIKYcdtszJa2XtCoifm97SvNFxKik0WoZ0UuTAOqb0qU321/QeNB/GhGvVJMP255b1edKOtKfFgE0wRHlna3Hd+HrJB2LiFUTpj8t6X8j4inbqyXNiYi/77Ks83bPftlll3WsvfHGG8V5FyxYUGvdp0+fLtZfeOGFjrVjx44V53344YeL9dmzyxdZbr755mJ9165dxTqaFxGTHnZP5TB+qaS/lfSu7W3VtEclPSXp57YfkLRf0jeaaBRAf3QNe0T8h6ROJ+hfbbYdAP3C12WBJAg7kARhB5Ig7EAShB1Iout19kZXdh5fZy+5//77i/XnnnuuWO92i+swu/7664t1bnEdvE7X2dmzA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAS/JR0A1566aVa899zzz3F+rJly2otH5DYswNpEHYgCcIOJEHYgSQIO5AEYQeSIOxAEtzPPgSmTZtWrI+MlEfPWrFiRcfa8uXLe+rprG5DOj/22GPF+qlTp2qtH58f97MDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJTGZ99nqSfSPpjSZ9IGo2If7H9uKQHJf1P9dZHI2JDl2VxnR3os07X2acS9rmS5kbEVtuzJL0t6S5J90g6ERHlb118elmEHeizTmGfyvjshyQdqp4ft71b0nXNtgeg3z7XObvt+ZIWSdpSTXrI9ju2X7Q9u8M8I7bHbI/V6hRALVP+brztmZL+XdL3I+IV29dIOiopJD2p8UP9b3dZBofxQJ/1fM4uSba/IOlXkn4dEf88SX2+pF9FxJ91WQ5hB/qs5xthbFvSjyTtnhj06oO7s74uaUfdJgH0z1Q+jf+KpDckvavxS2+S9KiklZIWavwwfp+k71Qf5pWWxZ4d6LNah/FNIexA/3E/O5AcYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IImuPzjZsKOS/nvC6yuracNoWHsb1r4keutVk739SafCQO9n/8zK7bGIWNxaAwXD2tuw9iXRW68G1RuH8UAShB1Iou2wj7a8/pJh7W1Y+5LorVcD6a3Vc3YAg9P2nh3AgBB2IIlWwm57ue09tt+3vbqNHjqxvc/2u7a3tT0+XTWG3hHbOyZMm2P7ddvvVY+TjrHXUm+P2/5dte222b6jpd7m2f6N7d22d9r+XjW91W1X6Gsg223g5+y2p0naK+lrkg5IekvSyojYNdBGOrC9T9LiiGj9Cxi2/0rSCUk/OTu0lu1/knQsIp6q/qOcHRH/MCS9Pa7POYx3n3rrNMz4t9Titmty+PNetLFnv0XS+xHxQUT8QdLPJN3ZQh9DLyI2Szp2zuQ7Ja2rnq/T+D+WgevQ21CIiEMRsbV6flzS2WHGW912hb4Goo2wXyfptxNeH9Bwjfcekl6z/bbtkbabmcQ1Z4fZqh6vbrmfc3UdxnuQzhlmfGi2XS/Dn9fVRtgnG5pmmK7/LY2Iv5D0N5K+Wx2uYmp+KOl6jY8BeEjSD9psphpmfL2kVRHx+zZ7mWiSvgay3doI+wFJ8ya8/qKkgy30MamIOFg9HpH0C42fdgyTw2dH0K0ej7Tcz/+LiMMRcSYiPpG0Vi1uu2qY8fWSfhoRr1STW992k/U1qO3WRtjfknSD7S/Zni7pm5JebaGPz7A9o/rgRLZnSFqm4RuK+lVJ91XP75P0yxZ7+ZRhGca70zDjannbtT78eUQM/E/SHRr/RP6/JP1jGz106OvLkrZXfzvb7k3Syxo/rDul8SOiByRdIWmTpPeqxzlD1Nu/anxo73c0Hqy5LfX2FY2fGr4jaVv1d0fb267Q10C2G1+XBZLgG3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kMT/ASoxHZh9Vl43AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_test(np.random.randint(test_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 激活函数的求导\n",
    "def d_softmax(data):\n",
    "    sm = softmax(data)\n",
    "    return np.diag(sm) - np.outer(sm, sm)\n",
    "# def d_tanh(data):\n",
    "#     return np.diag(1 / (np.cosh(data)) ** 2)\n",
    "def d_tanh(data):\n",
    "    return 1 / (np.cosh(data)) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.18256393 -0.03653857 -0.07772792 -0.06829743]\n",
      "[[ 0.18255919 -0.03653763 -0.0777259  -0.06829566]\n",
      " [-0.03653763  0.12892805 -0.04917855 -0.04321187]\n",
      " [-0.0777259  -0.04917855  0.21882837 -0.09192392]\n",
      " [-0.06829566 -0.04321187 -0.09192392  0.20343145]]\n",
      "[-0.03135162  0.15715605 -0.06751697 -0.05828746]\n",
      "[[ 0.13476328 -0.03135067 -0.05549968 -0.04791293]\n",
      " [-0.03135067  0.15715126 -0.06751491 -0.05828569]\n",
      " [-0.05549968 -0.06751491  0.22619699 -0.1031824 ]\n",
      " [-0.04791293 -0.05828569 -0.1031824   0.20938102]]\n",
      "[-0.07727372 -0.04762746  0.16900851 -0.04410733]\n",
      "[[ 0.2300435  -0.07931714 -0.07727153 -0.07345483]\n",
      " [-0.07931714  0.17221694 -0.0476261  -0.04527369]\n",
      " [-0.07727153 -0.0476261   0.1690037  -0.04410607]\n",
      " [-0.07345483 -0.04527369 -0.04410607  0.1628346 ]]\n",
      "[-0.04696228 -0.06533061 -0.05737506  0.16966794]\n",
      "[[ 0.16982447 -0.06541466 -0.05744887 -0.04696095]\n",
      " [-0.06541466  0.21066223 -0.07991882 -0.06532876]\n",
      " [-0.05744887 -0.07991882  0.19474112 -0.05737343]\n",
      " [-0.04696095 -0.06532876 -0.05737343  0.16966313]]\n"
     ]
    }
   ],
   "source": [
    "# 用导数定义检验softmax的对角矩阵/张量外积结果的第一行等于求导结果\n",
    "h = 0.0001\n",
    "func = softmax\n",
    "input_len = 4\n",
    "differential = {softmax: d_softmax, tanh: d_tanh}\n",
    "for i in range(input_len):\n",
    "    test_input = np.random.rand(input_len)\n",
    "    derivative = differential[func](test_input)\n",
    "    value1 = func(test_input)\n",
    "    test_input[i] += h\n",
    "    value2 = func(test_input)\n",
    "    print((value2 - value1) / h)\n",
    "    print(derivative)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将dimensions[-1]（输出层大小）转化成独热模式\n",
    "onehot = np.identity(dimensions[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算损失\n",
    "def sqr_loss(img, lab, parameters):\n",
    "    # 先预测\n",
    "    y_pred = predict(img, parameters)\n",
    "    # y = 当前标签的独热模式值\n",
    "    y = onehot[lab]\n",
    "    diff = y - y_pred\n",
    "    return np.dot(diff, diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8082815796798335"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sqr_loss(train_img[0], train_lab[0], parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 反向传播链式求导步骤：  \n",
    "![反向传播链式求导过程](./bp.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 求每层每个参数的梯度\n",
    "def grad_parameters(img, lab, parameters):\n",
    "    l0_in = img + parameters[0]['b']\n",
    "    l0_out = activation[0](l0_in)\n",
    "    l1_in = np.dot(l0_out, parameters[1]['w']) + parameters[1]['b']\n",
    "    l1_out = activation[1](l1_in)\n",
    "    \n",
    "    diff = onehot[lab] - l1_out\n",
    "    act1 = np.dot(differential[activation[1]](l1_in), diff)\n",
    "    grad_b1 = -2 * act1\n",
    "    grad_w1 = -2 * np.outer(l0_out, act1)\n",
    "    grad_b0 = -2 * differential[activation[0]](l0_in) * np.dot(parameters[1]['w'], act1)\n",
    "    \n",
    "    return {'w1': grad_w1, 'b1': grad_b1, 'b0': grad_b0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'w1': array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]),\n",
       " 'b1': array([ 0.02242527,  0.02708666,  0.01153643,  0.00584276, -0.17502686,\n",
       "         0.02435536,  0.03172234,  0.01411193,  0.02911151,  0.0088346 ]),\n",
       " 'b0': array([ 5.70284649e-03,  7.60792661e-03,  1.66384721e-03, -9.81477131e-03,\n",
       "        -1.27128257e-02, -1.13687824e-02, -6.69293420e-03,  3.89957343e-03,\n",
       "        -1.79803544e-02,  4.28178380e-03,  1.10244706e-02,  5.62128503e-03,\n",
       "        -3.08636559e-03,  9.88977783e-03,  4.40134125e-03,  9.97522090e-03,\n",
       "         1.00123424e-02, -9.42332819e-03,  1.09219723e-02, -1.60747447e-02,\n",
       "         9.12796204e-03, -5.57568915e-03,  5.71619896e-03, -4.41126871e-03,\n",
       "         1.13895431e-02,  5.08214948e-03, -1.10968603e-02, -1.09836085e-02,\n",
       "        -1.98764384e-02, -4.45067234e-03,  5.39969082e-03,  7.24532087e-03,\n",
       "        -1.21248213e-02,  1.61426668e-02, -8.46626434e-03,  8.53271874e-03,\n",
       "         3.49673381e-03, -1.42707834e-02, -5.73134510e-03,  9.03455523e-03,\n",
       "         2.13776419e-03,  8.82594524e-03, -1.07151749e-03, -3.80309939e-03,\n",
       "        -1.20394581e-02,  1.57316707e-02, -6.98810865e-03,  1.34370679e-02,\n",
       "        -2.71094030e-03, -1.08923022e-02,  7.99444653e-03, -1.72117864e-02,\n",
       "         1.48446816e-02, -7.79937061e-03, -2.09935911e-02,  6.88534498e-03,\n",
       "        -1.08509347e-02, -4.74307650e-03, -4.93291063e-04, -6.46910482e-03,\n",
       "        -8.56668364e-03, -1.05872815e-02,  1.30265303e-02,  4.82622521e-03,\n",
       "        -8.09117840e-03,  1.16415539e-02, -6.68758502e-03,  9.28366897e-03,\n",
       "        -8.39111852e-03,  2.84340448e-03,  1.32061629e-02,  4.78190328e-03,\n",
       "         3.42021039e-03,  1.43854216e-02,  1.93555550e-02,  1.52216138e-02,\n",
       "        -4.83461956e-03,  4.42016472e-03, -2.93539262e-03, -6.42972683e-03,\n",
       "         5.70236354e-03, -5.62897768e-04, -5.53957314e-03, -5.78000235e-03,\n",
       "        -1.06861605e-02,  1.51980398e-03, -2.34472499e-03,  5.61320485e-03,\n",
       "        -2.69127315e-03, -2.28260893e-02, -1.14818103e-02,  6.41720768e-03,\n",
       "         3.53753874e-03,  8.86287858e-03, -1.03659073e-02,  1.19155561e-02,\n",
       "        -1.29329611e-02, -1.19068856e-02,  6.78255676e-03, -9.93959195e-03,\n",
       "         1.50513433e-02,  8.27585281e-03,  1.72187853e-02,  3.27016810e-03,\n",
       "         5.10785110e-03, -5.29582398e-03,  1.22503615e-02, -8.18660241e-03,\n",
       "         1.71891130e-02, -1.45103403e-02, -7.07966063e-03, -4.90485456e-03,\n",
       "        -1.69964620e-03,  9.60513086e-03, -1.05834362e-02, -1.09348593e-02,\n",
       "         9.97583992e-03, -1.23583853e-02, -7.16051356e-03, -5.85024667e-03,\n",
       "        -6.12616094e-03, -1.60131848e-02,  1.27967194e-03,  1.17283533e-02,\n",
       "         1.64429863e-02,  1.61901004e-02,  1.03263068e-02, -1.43246503e-02,\n",
       "         8.52428377e-03,  5.17676145e-04, -6.90262780e-03,  1.08852454e-02,\n",
       "         4.96348445e-03, -1.31216901e-02,  9.37328330e-03, -1.38605509e-03,\n",
       "         5.63741386e-03, -1.57905021e-02, -1.45932449e-02, -1.70915069e-03,\n",
       "         3.71721335e-03,  1.35011591e-02, -5.68086621e-03, -5.21376852e-03,\n",
       "        -1.00878376e-02, -3.61262295e-03, -6.47481033e-03, -7.54843243e-03,\n",
       "        -4.82636770e-03,  2.27238228e-03, -7.32567703e-03,  1.70720167e-03,\n",
       "         5.02508824e-03,  9.25953922e-03, -5.14439639e-03,  1.13863668e-02,\n",
       "         5.05286564e-03, -6.15807212e-03,  6.58533117e-03, -4.64509160e-03,\n",
       "         1.98347212e-03, -4.67870786e-03, -1.34399614e-02, -1.39571871e-02,\n",
       "        -6.14936779e-05,  3.66278983e-03,  1.16007571e-02, -6.91326272e-03,\n",
       "         1.23463338e-02, -1.03691231e-02,  7.60348502e-03,  3.42102417e-03,\n",
       "         4.12558693e-03,  1.02011548e-02,  2.14515737e-03,  9.71111797e-03,\n",
       "        -4.94598333e-04,  3.57698008e-03,  1.08837914e-02,  7.43019859e-04,\n",
       "        -3.83936090e-03, -8.56902990e-03, -2.72838005e-03,  1.38243237e-02,\n",
       "         1.36079601e-02, -5.78907422e-03,  3.97275127e-03, -1.35911615e-02,\n",
       "        -2.28413266e-03, -3.79515149e-03,  1.03095483e-02, -1.13374018e-02,\n",
       "         2.15131347e-03,  1.32226614e-02,  1.05962300e-02, -3.11593327e-03,\n",
       "        -2.03268983e-03,  1.35040939e-02, -3.40734251e-03, -6.56507632e-03,\n",
       "        -3.86396549e-03,  8.73274622e-03,  5.63999749e-03, -1.23034627e-02,\n",
       "         1.02460321e-02, -1.74354255e-02, -2.67950108e-04,  1.14722513e-02,\n",
       "         5.22955447e-03,  6.89152433e-03, -7.77200714e-03,  1.69507129e-03,\n",
       "        -1.82819610e-02,  1.50794086e-02,  7.30072665e-03,  8.40485414e-03,\n",
       "         9.49686854e-03,  8.72669960e-03, -1.60171375e-02,  3.79806964e-04,\n",
       "        -1.10862875e-02,  1.50101456e-03, -7.05063909e-03,  1.75389035e-02,\n",
       "         3.86932606e-03,  1.34778437e-02, -9.78515442e-04, -1.39176963e-02,\n",
       "         1.63533261e-03, -2.78655250e-03, -2.96275670e-03,  5.95574177e-03,\n",
       "         2.92903754e-03, -6.57044308e-03, -7.26127300e-03, -1.04797691e-02,\n",
       "         7.56100615e-03, -1.22131998e-04, -8.46264674e-03,  1.13511748e-04,\n",
       "         1.84674674e-03,  6.37959652e-03,  1.12021582e-02,  9.41533749e-03,\n",
       "        -3.15251680e-03, -7.68552499e-03, -8.41696869e-03,  1.92533856e-03,\n",
       "         1.49420830e-02,  4.39211393e-03, -1.55082278e-03, -1.74418793e-02,\n",
       "         2.20865386e-03,  1.75447468e-02,  9.73444914e-03, -1.43817260e-02,\n",
       "         2.63967647e-03, -7.77630747e-03, -4.00631936e-03, -1.97269099e-02,\n",
       "         3.20249349e-03,  6.28345625e-03,  4.44092165e-03, -9.85970776e-03,\n",
       "         1.20257043e-02, -1.10708028e-02, -9.10280955e-03, -1.37330131e-02,\n",
       "         1.09157544e-03,  3.83995130e-03,  1.43605957e-02, -4.84650790e-03,\n",
       "        -6.36416122e-03,  1.03388920e-02, -1.32348987e-02, -1.46608875e-02,\n",
       "         6.54235299e-03,  1.70413077e-03,  3.21113417e-04, -1.07551359e-02,\n",
       "        -2.34905298e-03, -1.25795521e-02, -4.75546358e-03, -1.01010551e-02,\n",
       "        -7.81450997e-03,  1.16183048e-03,  1.24827343e-02,  1.49587092e-02,\n",
       "         1.26620431e-02,  5.89353849e-03, -3.99076898e-03,  1.09111140e-02,\n",
       "        -1.60416308e-02,  3.24158297e-03,  1.65244626e-02,  9.85431288e-03,\n",
       "        -4.15982935e-03, -1.91088243e-02,  4.08569338e-03,  4.74638076e-03,\n",
       "        -4.73625541e-03,  1.34718324e-03,  1.27789332e-02,  6.75008288e-03,\n",
       "         1.00132345e-02,  1.23212635e-03, -2.71615648e-03, -1.55355596e-02,\n",
       "         4.18335732e-04, -4.52964509e-03,  4.41023669e-03,  4.66818879e-03,\n",
       "         6.98548159e-03, -1.03453554e-02, -1.23822140e-02, -1.17207676e-02,\n",
       "        -7.93598218e-03, -1.29495079e-02,  5.52158755e-03,  8.08439837e-04,\n",
       "         4.51974712e-03,  5.18114217e-03,  9.59033635e-03,  1.55099586e-02,\n",
       "        -1.17053920e-02,  1.21067536e-02, -1.70648181e-02,  2.04981676e-03,\n",
       "         2.99685386e-03,  1.48207533e-02,  8.98963696e-03, -8.62767464e-03,\n",
       "        -1.03454068e-02,  1.23383041e-02, -6.77424772e-03,  5.53330899e-03,\n",
       "        -5.12722489e-03,  5.62024660e-03,  1.17503030e-02, -2.21079936e-03,\n",
       "         2.41292988e-03,  6.16661212e-03, -5.32976739e-03, -6.01647356e-03,\n",
       "         2.16054398e-03, -2.62749067e-03,  1.25613507e-02, -1.08205802e-02,\n",
       "        -1.93730252e-03,  9.16929822e-03, -1.31633789e-02, -2.13710762e-03,\n",
       "         6.51601182e-03, -1.90830555e-04, -3.90318046e-03,  1.82556782e-03,\n",
       "         3.02475893e-04,  9.27412579e-03,  2.30390637e-04,  2.58515443e-03,\n",
       "         7.58783683e-03, -7.33240501e-03, -8.38709735e-03,  3.74554778e-03,\n",
       "        -2.66802899e-03, -6.84061153e-03,  2.63159842e-03,  7.87493087e-03,\n",
       "         3.34412046e-03,  5.88479947e-03, -1.70650342e-02, -2.00271696e-02,\n",
       "         3.06773626e-03,  2.25335278e-02, -3.67235624e-04, -1.17440125e-02,\n",
       "        -1.86585947e-02,  9.39414845e-03,  3.81914802e-03,  1.66237109e-02,\n",
       "        -8.03303678e-03, -2.25246345e-03, -8.84284666e-04, -2.07393462e-03,\n",
       "        -9.42089945e-04,  8.20713858e-03,  1.05501570e-02,  6.98576568e-03,\n",
       "         5.72611022e-03, -7.55061623e-03,  6.23857507e-04,  1.42213340e-02,\n",
       "         1.18998552e-02, -1.45983020e-02,  1.32826555e-02, -9.73265117e-04,\n",
       "         4.92139812e-06, -4.92204518e-03,  1.25037429e-02,  5.30216802e-03,\n",
       "        -7.93589168e-03,  8.03693831e-03,  2.22151613e-03,  1.07436730e-02,\n",
       "        -8.00000993e-03,  6.67933981e-03,  1.17514310e-03, -5.03058034e-03,\n",
       "         6.66057580e-03,  4.96092425e-03,  1.23357310e-03, -7.88409727e-03,\n",
       "        -7.94675963e-03,  2.78986971e-03,  1.15484805e-02, -1.78608894e-02,\n",
       "         1.05508064e-02, -7.15452489e-03,  1.39244025e-02,  3.40930337e-03,\n",
       "        -1.79359735e-03,  6.61439002e-03,  1.40063374e-02,  8.55151300e-03,\n",
       "         6.01181463e-03,  1.73936824e-03,  7.27626931e-04,  3.99856221e-04,\n",
       "        -2.40415507e-03, -5.79059082e-04,  3.58650210e-03,  4.53033901e-03,\n",
       "         4.04345373e-03,  1.24551633e-02, -3.33365720e-03,  5.79558008e-03,\n",
       "         8.84720540e-03, -1.19403750e-03, -6.44767856e-03, -6.02954638e-03,\n",
       "        -4.85744238e-03,  4.64361305e-04,  1.51412734e-02,  1.07670702e-02,\n",
       "        -8.81813591e-03, -7.05941638e-03, -6.25966257e-03, -1.64062911e-02,\n",
       "         1.46548371e-02,  1.39044978e-03, -6.29677611e-03,  1.45522359e-02,\n",
       "        -8.25731311e-03,  1.34248729e-03,  9.02905479e-03, -1.14873463e-02,\n",
       "         1.27116828e-02,  7.61839549e-03,  1.03023130e-03,  1.17339302e-03,\n",
       "        -1.27164610e-02, -1.34697057e-02,  3.50108135e-04, -3.33182321e-04,\n",
       "        -6.40369750e-03,  5.66176075e-03, -1.26300825e-03,  4.30513291e-03,\n",
       "         1.04203671e-02, -2.04775616e-02, -2.84642212e-03,  3.90083120e-03,\n",
       "         8.65614674e-03, -7.53802413e-03, -1.66742282e-02, -3.94897190e-03,\n",
       "        -1.14585290e-03, -1.87181355e-04, -3.26193339e-04,  1.77145647e-02,\n",
       "        -1.39674339e-02, -5.93457956e-03, -8.31555978e-03,  8.25685480e-03,\n",
       "         9.71391460e-03, -1.08247355e-02, -1.76254629e-02, -1.83143571e-02,\n",
       "         7.99927414e-03,  8.68617250e-03,  9.54786460e-03,  5.17676574e-03,\n",
       "         2.66844081e-03,  5.15271293e-03, -2.08650101e-03, -7.35331997e-03,\n",
       "         1.18787320e-02,  5.13267311e-04, -2.85097371e-03,  9.80940715e-03,\n",
       "         1.82262747e-02,  1.04065325e-02,  1.15299466e-02, -7.00502911e-03,\n",
       "         1.15490981e-02, -7.38995617e-05,  3.92591632e-03, -6.52215541e-03,\n",
       "         3.10687545e-04, -6.00142936e-03, -1.26147128e-02,  1.19386180e-02,\n",
       "         1.38965635e-02,  8.23103534e-03, -4.81050075e-03, -4.34338949e-03,\n",
       "        -4.24458699e-03, -3.08097416e-03,  3.30600886e-03, -1.13735386e-03,\n",
       "        -1.05148325e-02, -1.02530939e-02,  4.99069989e-03, -4.72603562e-03,\n",
       "        -1.76124928e-02, -8.30288034e-03, -3.32427653e-03,  7.12646996e-03,\n",
       "         7.89275922e-03, -1.59374635e-03, -4.48512590e-03, -7.00779043e-03,\n",
       "         1.77081118e-03,  1.32147155e-02, -1.09737637e-02, -1.13737232e-02,\n",
       "        -2.27591849e-03, -5.93639284e-03, -4.37741729e-04, -3.73678225e-03,\n",
       "         8.67393616e-03,  1.64515405e-03, -1.50665175e-02,  2.67917748e-03,\n",
       "        -6.28338645e-03,  9.25420870e-03, -8.53445755e-03, -2.39271152e-03,\n",
       "         1.23961441e-02, -3.61482419e-03, -3.28442309e-03,  1.73566107e-03,\n",
       "        -8.38873847e-03,  8.47832074e-04,  2.87170178e-03,  1.32145712e-02,\n",
       "        -1.26226100e-02, -1.78255906e-02, -1.06902484e-02,  3.81731768e-03,\n",
       "        -7.01232675e-03,  1.96184759e-02,  1.08444319e-02,  7.08464982e-03,\n",
       "         5.24422891e-03,  5.20249656e-03, -9.17059800e-03, -1.48954635e-02,\n",
       "        -4.16928352e-03,  2.14997523e-03, -1.60651719e-02,  8.37228191e-03,\n",
       "         1.08193082e-02,  1.00867849e-02,  6.12230486e-05, -4.52855863e-03,\n",
       "        -1.15207290e-02, -1.05527246e-02,  7.40670697e-03, -3.32540267e-03,\n",
       "         1.26286211e-02,  6.62640767e-03,  4.92603244e-03,  1.91584478e-02,\n",
       "        -2.74558934e-04,  1.07572221e-02,  5.98725700e-03, -1.01195137e-02,\n",
       "        -1.39974040e-02, -1.16344301e-02,  2.41525506e-03,  4.28161394e-03,\n",
       "        -7.16039282e-03, -4.14775574e-05, -1.47812078e-02,  1.81147788e-02,\n",
       "         3.03157232e-03,  9.23106812e-03, -1.01639115e-02,  8.27667192e-03,\n",
       "        -6.96212300e-03, -5.67620092e-03, -1.24545893e-02,  3.60050415e-04,\n",
       "         1.82809978e-02,  6.10796126e-04, -5.01196771e-03, -3.85718369e-03,\n",
       "         5.53932617e-03, -3.47122449e-05,  1.62432026e-02, -3.91279585e-03,\n",
       "        -5.02725909e-03, -1.28339301e-02,  4.79980205e-03, -7.25198292e-03,\n",
       "        -5.10032830e-03,  1.37035637e-02,  5.54901999e-03,  5.04715526e-03,\n",
       "        -1.45519885e-02,  9.70320550e-03,  3.12052525e-03, -1.29287010e-02,\n",
       "        -1.16265453e-02, -3.90285311e-03, -7.43843754e-03, -6.71265898e-03,\n",
       "         9.06986031e-03,  4.88080430e-03,  1.25570410e-02, -1.90003433e-03,\n",
       "         7.60144181e-03,  2.76512451e-04,  1.21825613e-03, -9.22851616e-03,\n",
       "        -1.26295397e-02,  7.14493531e-03,  1.53267919e-02,  1.53307142e-02,\n",
       "        -3.23734245e-04,  4.98241346e-03,  5.21443591e-03,  7.14238661e-03,\n",
       "        -1.35577221e-02, -1.64058643e-02,  1.91252954e-03, -4.62045440e-03,\n",
       "         1.01543730e-02, -2.54725169e-03,  1.56009154e-02,  1.70357246e-03,\n",
       "         6.24312702e-04,  5.09408225e-03, -3.23354152e-03,  1.75198917e-02,\n",
       "        -3.15234671e-03, -2.68039029e-03, -6.40992514e-03,  1.32959447e-02,\n",
       "        -1.38425167e-02,  2.08075301e-03, -1.06419205e-03, -1.50283650e-03,\n",
       "        -1.08483144e-02, -1.00483449e-02,  9.09686039e-03,  1.38930878e-02,\n",
       "         9.92994324e-03,  1.05834318e-02, -1.14750042e-02,  7.39277783e-03,\n",
       "         6.26230385e-03,  7.29781320e-03,  5.22575251e-03, -1.30522896e-02,\n",
       "         5.09694605e-03,  7.78007726e-03, -3.54922289e-03, -2.57055715e-03,\n",
       "         8.97039685e-04, -1.15187137e-03,  1.17585835e-02, -6.72158369e-03,\n",
       "        -3.75827552e-03,  4.06896238e-03,  8.82155974e-03,  1.44673104e-02,\n",
       "         9.41043798e-04, -3.47449957e-04, -3.06759342e-03,  6.30151188e-03,\n",
       "        -4.83812221e-03,  1.15290813e-02,  5.44860139e-03, -3.98979322e-03,\n",
       "         1.15842226e-02, -1.47168716e-02, -1.13435875e-02,  8.64743990e-03,\n",
       "        -9.72565201e-03, -7.95045178e-03, -7.32695821e-03, -1.31078038e-02,\n",
       "        -4.78034485e-03,  6.04745067e-03, -2.89423393e-04,  1.11282868e-02,\n",
       "         2.21252985e-03,  5.64890244e-03,  6.30559695e-03, -1.60730093e-03,\n",
       "        -4.74115344e-03,  2.34314377e-03,  8.35818977e-03, -2.12643523e-03,\n",
       "         1.09903087e-02,  3.45306911e-03,  2.30672536e-03,  4.22815205e-03,\n",
       "         1.25572499e-02, -7.28270018e-04,  1.04485427e-03, -1.74116835e-03,\n",
       "        -1.07616751e-04, -8.09920634e-03, -9.88612795e-03, -1.03812470e-02,\n",
       "        -1.19601658e-02, -5.25331860e-03,  6.06690408e-03, -8.60949590e-03,\n",
       "         9.45634616e-04, -1.03968294e-02, -1.90240615e-02,  4.08817973e-03,\n",
       "        -8.81470764e-03,  1.17343177e-02,  2.21361816e-04,  9.58846966e-04,\n",
       "         2.73077637e-03, -8.05024647e-03, -1.40037182e-02,  1.47077435e-02,\n",
       "        -4.41201068e-03, -4.61254089e-03, -4.67839327e-03,  5.69053280e-03,\n",
       "        -3.58877425e-03, -1.09848315e-02, -1.05025792e-02, -9.77560916e-03,\n",
       "        -6.58363733e-03, -1.69240230e-03,  4.46181012e-03, -1.13564735e-02,\n",
       "         2.09680443e-03, -8.39727347e-03,  4.18348101e-03, -1.48299571e-02,\n",
       "        -1.70951600e-02,  9.98830076e-03, -1.41829039e-02,  1.16691407e-02,\n",
       "         7.27640850e-04,  1.65574771e-02,  1.10129380e-04,  1.16184215e-02,\n",
       "        -7.94197411e-03, -5.21477349e-03,  7.93461507e-03,  8.05447237e-03,\n",
       "        -7.99955933e-03, -8.20785955e-03, -3.62618530e-03, -6.24662593e-03,\n",
       "        -2.51834568e-03, -3.96573271e-03,  1.04320909e-02,  4.94910069e-03,\n",
       "        -1.23958907e-02,  1.89954853e-02,  1.17633475e-03, -5.52580668e-04])}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_parameters(train_img[2], train_lab[2], init_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.16439866620216e-08\n",
      "-3.53116036480583e-08\n",
      "-3.188475786869427e-08\n",
      "-1.537275861121512e-08\n",
      "-1.8018792333207578e-08\n",
      "-9.842569530238432e-09\n",
      "-8.609349158063273e-09\n",
      "-1.4872970979795674e-08\n",
      "-7.811280091502004e-09\n",
      "5.5444727337095046e-08\n"
     ]
    }
   ],
   "source": [
    "# 验证b求导公式精确性\n",
    "h = 0.000001\n",
    "for i in range(10):\n",
    "    img_i = np.random.randint(train_num)\n",
    "    test_parameters = init_parameters()\n",
    "    derivative = grad_parameters(train_img[img_i], train_lab[img_i], test_parameters)['b1']\n",
    "    \n",
    "    value1 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)\n",
    "    test_parameters[1]['b'][i] += h\n",
    "    value2 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)\n",
    "    print(derivative[i] - (value2 - value1) / h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 验证集的损失函数\n",
    "def valid_loss(parameters):\n",
    "    loss_accu = 0\n",
    "    for img_i in range(valid_num):\n",
    "        loss_accu += sqr_loss(valid_img[img_i], valid_lab[img_i], parameters)\n",
    "    return loss_accu\n",
    "# 验证集的准确度\n",
    "def valid_accuracy(parameters):\n",
    "    correct = [predict(valid_img[img_i], parameters).argmax() == valid_lab[img_i] for img_i in range(valid_num)]\n",
    "    accuracy = correct.count(True) / len(correct)\n",
    "    print(f'validation accuracy: {accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9039.409425641368"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_loss(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation accuracy: 0.1186\n"
     ]
    }
   ],
   "source": [
    "valid_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 分批训练，一批100张图片\n",
    "batch_size = 100\n",
    "def train_batch(current_batch, parameters):\n",
    "    grad_accu = grad_parameters(train_img[current_batch * batch_size + 0], train_lab[current_batch * batch_size + 0], parameters)\n",
    "    for img_i in range(1, batch_size):\n",
    "        # 每批求一次损失和，然后取平均\n",
    "        grad_tmp = grad_parameters(train_img[current_batch * batch_size + img_i], train_lab[current_batch * batch_size + img_i], parameters)\n",
    "        for key in grad_accu.keys():\n",
    "            grad_accu[key] += grad_tmp[key]\n",
    "    # 取平均过程\n",
    "    for key in grad_accu.keys():\n",
    "        grad_accu[key] /= batch_size\n",
    "    # 返回平均梯度\n",
    "    return grad_accu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'w1': array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]),\n",
       " 'b1': array([-0.0051357 , -0.01172677,  0.02510991, -0.00288452, -0.00552647,\n",
       "         0.00048272,  0.00187308, -0.00285783,  0.00104385, -0.00037826]),\n",
       " 'b0': array([ 2.91449029e-03,  1.66564380e-03,  6.26498043e-04, -1.31873113e-03,\n",
       "         1.84675575e-03, -8.61275600e-04,  3.10776173e-04,  1.45655069e-03,\n",
       "         6.12233093e-04, -1.01829505e-03, -1.23751105e-04,  8.23014443e-04,\n",
       "        -1.07091086e-03, -1.78556849e-04, -1.48873966e-03, -1.73522836e-03,\n",
       "        -2.59304625e-03,  4.36514762e-04,  1.96832644e-03, -3.72384303e-04,\n",
       "        -1.41126699e-03, -3.03673547e-03,  1.81980190e-03, -2.67172464e-03,\n",
       "         1.34914623e-03, -1.93098720e-03, -6.26524558e-04, -1.76767856e-03,\n",
       "        -2.41316103e-04, -1.57151594e-03,  1.91893889e-03,  2.70550864e-04,\n",
       "        -6.92459953e-04, -1.98855174e-03,  1.82877612e-03,  8.29583255e-04,\n",
       "        -9.59471541e-04,  2.43509284e-03, -1.13251167e-03,  1.25711836e-03,\n",
       "        -2.37232073e-03,  7.66188612e-04, -1.45689715e-03, -1.59694707e-03,\n",
       "        -2.05161323e-03, -4.77723535e-04, -7.25013709e-04, -7.07725487e-04,\n",
       "         2.68589196e-03,  1.67845207e-04,  1.13623566e-04, -2.64921757e-04,\n",
       "        -2.16363149e-03, -1.57166557e-03, -1.13825155e-03, -1.68295470e-03,\n",
       "         1.62227622e-03,  1.19356745e-03,  4.51764835e-04, -1.38501993e-03,\n",
       "         1.19708962e-03, -9.64702615e-04,  1.69028092e-03,  1.09491420e-03,\n",
       "        -2.04083468e-03,  4.35763704e-04,  9.84686536e-04, -2.61917146e-04,\n",
       "        -2.62831690e-04, -3.67547843e-04,  8.25481893e-04,  1.56887003e-05,\n",
       "         8.90011253e-04, -7.41401660e-04,  2.05252556e-03, -5.62701638e-04,\n",
       "         1.92222233e-03, -7.18647404e-04, -8.59276415e-04, -3.24003851e-04,\n",
       "         1.62807667e-03,  1.44734069e-03, -6.89459073e-04,  9.62043193e-04,\n",
       "         1.07635029e-03,  3.82706163e-04,  2.38925945e-03, -2.04233655e-03,\n",
       "         2.53143517e-03,  2.74019022e-03,  1.84142861e-03,  3.15497779e-03,\n",
       "         1.87189262e-03,  1.18976661e-03, -7.52381428e-04,  1.32025070e-03,\n",
       "        -2.42920579e-03,  9.62503564e-04, -3.69585852e-04, -4.00565962e-04,\n",
       "         2.61742645e-03, -1.58836484e-03,  1.96943278e-04, -1.50889511e-03,\n",
       "        -2.23491457e-03,  1.89388120e-03,  7.52180743e-04, -3.71556592e-04,\n",
       "         1.20936690e-03, -4.60202790e-04,  2.33736447e-03,  8.64202992e-04,\n",
       "        -6.09922594e-04, -1.24140753e-03, -2.27797753e-03,  1.34327403e-03,\n",
       "         1.30470963e-03,  2.51757198e-03,  2.93660613e-03,  7.38675214e-04,\n",
       "         7.45684502e-04,  3.13256877e-03, -1.79795445e-04,  3.63624161e-04,\n",
       "         1.88095474e-03, -2.49090766e-04, -2.85789246e-04,  1.60424194e-03,\n",
       "        -7.20618722e-04,  1.37363169e-03,  1.31817496e-03,  5.28366763e-05,\n",
       "         2.17997831e-03,  3.41703306e-03,  2.96382244e-03,  8.79546298e-04,\n",
       "        -1.49454773e-03, -1.44893175e-04, -3.31622271e-04, -4.40792432e-04,\n",
       "         9.21168329e-04,  1.09258339e-03,  1.41954713e-03,  1.03461030e-03,\n",
       "        -6.65341552e-05, -1.54479737e-03, -1.50116053e-03,  1.07171637e-03,\n",
       "         1.17111918e-03,  8.75802395e-04, -6.38839811e-04,  4.03294655e-04,\n",
       "        -3.70594634e-03,  2.39593150e-03,  1.50948027e-03,  1.07747075e-03,\n",
       "         2.54234513e-04,  6.15953935e-04,  2.09160439e-03, -2.01187364e-04,\n",
       "         1.11655796e-03,  1.64755086e-03, -1.55055846e-03,  1.60077585e-03,\n",
       "        -4.86351905e-04, -4.14082679e-04,  4.44083122e-04,  5.78821069e-04,\n",
       "         3.96838650e-04,  1.90174166e-03, -4.20657928e-04, -4.76394872e-04,\n",
       "        -1.34286423e-03, -1.30589772e-03,  9.67217132e-04, -1.56646884e-03,\n",
       "        -5.28803637e-05,  2.16330582e-03,  3.02467694e-03, -1.02752306e-03,\n",
       "         2.43662993e-03,  1.53547458e-03,  1.86731783e-03, -3.32596680e-03,\n",
       "        -9.15223026e-04,  2.47059843e-03, -6.72967076e-04,  1.09610161e-03,\n",
       "        -1.20602744e-03, -1.19851123e-03,  7.71116045e-04,  2.61606726e-04,\n",
       "        -1.33927187e-03,  2.50896375e-04,  8.51264113e-04, -8.29506735e-04,\n",
       "        -7.76379123e-04,  3.35383930e-03, -3.67915232e-04,  1.68106017e-03,\n",
       "         1.07534649e-03, -6.13500250e-04,  1.58715848e-03, -8.01548707e-04,\n",
       "         1.98747484e-03, -1.67663491e-03, -1.69031990e-03,  7.71907131e-04,\n",
       "        -1.05473530e-03,  2.71883073e-03, -8.63846998e-04, -3.93976393e-04,\n",
       "         7.29620915e-04,  4.03397976e-04, -2.72718511e-05, -2.91670762e-03,\n",
       "         9.49798445e-04,  3.06997697e-04, -6.33495168e-04,  3.47942182e-04,\n",
       "        -4.47365622e-04,  2.58276289e-03, -1.21058409e-03, -9.17709552e-04,\n",
       "         1.91837430e-03, -5.32403928e-04, -2.40404621e-03,  9.47265711e-04,\n",
       "        -2.97513199e-04, -9.09675177e-04,  1.01710705e-03, -1.53703516e-03,\n",
       "        -2.15763994e-03,  9.68382191e-04,  7.26796061e-04, -1.27711646e-03,\n",
       "        -7.19377870e-04,  1.29675967e-03,  1.49957590e-03,  4.68669025e-05,\n",
       "        -2.95166322e-04,  1.39103799e-03, -1.60186248e-04,  6.15568331e-04,\n",
       "         5.47905699e-04, -1.47165094e-04,  2.22760822e-03,  1.00774514e-03,\n",
       "         1.44296342e-03,  5.43340722e-04, -1.55632145e-04,  1.28784519e-03,\n",
       "        -2.58280002e-03,  1.38832090e-03, -4.15447228e-04,  1.20195069e-03,\n",
       "        -5.36458150e-04, -1.58384161e-03, -2.20256663e-03, -2.06772406e-03,\n",
       "        -3.58468810e-04,  7.76868735e-04, -9.66582728e-04,  1.17003879e-03,\n",
       "         2.58736420e-04,  5.38849414e-04,  5.31601787e-06,  9.04952113e-04,\n",
       "         7.63611570e-04, -1.53907494e-03,  1.85330141e-04,  4.66103418e-04,\n",
       "         4.39774296e-04,  1.34133085e-03, -6.07759136e-04,  5.40348269e-04,\n",
       "         1.23468339e-03,  2.66224597e-03, -9.95228739e-04,  1.08269007e-03,\n",
       "         6.76308356e-04, -1.11620039e-03,  5.07502194e-04, -1.42515490e-03,\n",
       "        -1.23347745e-03, -2.79314376e-04, -3.76748672e-05, -1.85288080e-04,\n",
       "         1.48097097e-03,  1.44367284e-03,  2.44728145e-03, -8.41382157e-04,\n",
       "        -1.63635167e-03,  1.10525754e-04,  1.23208974e-03,  1.72816193e-03,\n",
       "        -3.40130508e-04, -1.70169077e-03,  1.21383360e-03, -1.61377495e-03,\n",
       "         1.01418671e-03, -1.90974178e-03, -1.05366030e-03,  3.46961842e-04,\n",
       "        -1.19345914e-03,  1.64401412e-03, -1.84434614e-04, -3.00518365e-04,\n",
       "        -1.37054349e-03,  1.71262252e-03, -1.50020012e-03, -7.77727707e-04,\n",
       "        -8.08759819e-04, -1.11110001e-03, -2.56967901e-04,  1.86880840e-03,\n",
       "         8.80800977e-04, -1.99962178e-04, -1.25779532e-04,  1.52150319e-03,\n",
       "         8.60334454e-06,  6.15068541e-04, -1.18379066e-03,  7.97362399e-04,\n",
       "         9.97993717e-04,  1.43446588e-03, -2.13012182e-03, -4.07933124e-04,\n",
       "         2.22351731e-04, -6.30548185e-04,  9.81284026e-04, -2.01706932e-03,\n",
       "         8.24161482e-04,  1.64576189e-03, -7.32792944e-04,  7.97265402e-04,\n",
       "         6.83863047e-04,  6.31947207e-04, -2.19874582e-03,  1.20559403e-03,\n",
       "         3.27276734e-04, -3.65092716e-04, -3.49821912e-04, -1.94338210e-03,\n",
       "         7.43904019e-04,  4.15568275e-04, -4.60098944e-04, -1.02602740e-03,\n",
       "         8.29015306e-04, -1.30796985e-04,  5.69607506e-04, -2.11937017e-03,\n",
       "         9.41382770e-04,  1.41005850e-03,  9.23620091e-04, -3.87788778e-04,\n",
       "        -9.66919956e-04,  1.53467701e-04,  1.99601464e-03,  1.04854268e-03,\n",
       "         1.44613187e-04,  4.58520820e-05, -1.27881495e-03,  5.20689894e-04,\n",
       "         1.42593347e-03, -2.21627554e-03, -1.56582785e-03,  8.76904328e-04,\n",
       "        -1.55681683e-03, -8.99236288e-04, -8.02473136e-04,  1.32312386e-03,\n",
       "        -5.19208222e-04, -1.18743623e-03, -2.62795350e-03,  7.97399181e-04,\n",
       "        -1.20333943e-03,  6.82024717e-04, -8.76164914e-04,  2.91004560e-04,\n",
       "         6.50675560e-04,  1.31524052e-03, -9.44983355e-04,  1.37828099e-03,\n",
       "        -1.14155215e-04, -1.60744602e-03,  2.68646548e-03, -3.14973921e-03,\n",
       "         5.35204744e-04, -3.29845072e-03,  3.72822987e-04, -2.11384500e-03,\n",
       "        -2.24029492e-03, -3.72406823e-04, -2.67199451e-03,  2.34140391e-03,\n",
       "        -1.27568518e-03,  4.58300868e-04,  4.31669231e-04, -1.85816332e-03,\n",
       "        -9.96284787e-04,  2.43827800e-03,  1.34941743e-04,  1.71185327e-03,\n",
       "        -5.17662670e-05,  8.80977436e-04, -1.30125914e-03,  6.88511696e-04,\n",
       "        -1.48030907e-03,  1.48482822e-03,  5.26659572e-04, -1.31425693e-03,\n",
       "        -2.06455223e-05, -1.02180884e-03,  1.41563064e-03, -2.59447434e-03,\n",
       "        -1.62661191e-03, -5.73232862e-05, -1.32901279e-03, -3.02932796e-04,\n",
       "         2.26610835e-03,  5.21901255e-04,  2.76566956e-03, -7.54975102e-04,\n",
       "        -9.17257598e-04,  4.68767892e-04,  2.06604926e-03,  2.41918104e-03,\n",
       "        -1.40089485e-03, -1.68636302e-04,  4.92872377e-04,  1.19070983e-03,\n",
       "         1.02871689e-03,  4.53896888e-04, -7.81412839e-04, -9.66798791e-04,\n",
       "         3.08858789e-04,  7.43023194e-05,  1.31299193e-03, -6.62136329e-04,\n",
       "        -2.39192691e-04,  3.38215861e-04, -1.52726513e-04,  1.35057304e-03,\n",
       "        -9.37113396e-04, -4.90056338e-04, -8.60570147e-04, -9.04132529e-04,\n",
       "         8.47336673e-05,  1.44917804e-03, -1.25433040e-03, -4.70491739e-04,\n",
       "        -2.67934871e-03, -1.23640873e-03, -1.06479293e-03, -9.24028457e-04,\n",
       "        -3.74948690e-04,  3.13981043e-04, -1.16809387e-03, -1.08604145e-03,\n",
       "         1.62681451e-04,  1.16742524e-03,  4.70399845e-05,  1.77185255e-03,\n",
       "        -5.79878708e-04,  1.02733587e-03,  1.02261977e-03, -9.15742239e-04,\n",
       "        -2.35973042e-03, -1.26432900e-03, -1.52534647e-03, -1.72539829e-03,\n",
       "         9.90496899e-04,  2.21256491e-03, -1.58114105e-03, -5.42855208e-04,\n",
       "         4.80656846e-04,  2.62065444e-04,  9.71662687e-04,  2.05454386e-03,\n",
       "        -2.03024005e-03, -1.50316120e-03, -1.27087668e-03,  3.48263084e-04,\n",
       "         1.19238062e-03, -3.72415395e-04,  3.34735417e-03, -2.21068755e-03,\n",
       "         9.13932221e-04,  1.72940058e-03,  1.22823971e-03, -4.66729858e-04,\n",
       "         5.10258139e-04, -1.86604418e-04,  5.58129248e-05,  2.03868964e-03,\n",
       "        -8.54540886e-04, -1.05203852e-03, -1.88629748e-03,  2.31793050e-03,\n",
       "         5.18587418e-04,  1.44719596e-03,  5.99351592e-04, -2.05592692e-04,\n",
       "         2.94574464e-03, -4.55843378e-04,  1.01433808e-03, -1.20426373e-04,\n",
       "         1.66839153e-04,  3.02313012e-03,  2.25182799e-03,  4.24634739e-04,\n",
       "        -2.71679649e-03,  3.13260036e-03, -9.54846275e-04,  2.30440165e-03,\n",
       "        -2.72271424e-03,  2.03086097e-03, -1.25203491e-03, -1.15566370e-03,\n",
       "        -1.10042301e-03,  5.05759288e-04,  6.10270205e-04,  3.29060458e-04,\n",
       "         7.25986542e-04,  1.91153260e-03,  1.52922079e-03, -1.07416733e-03,\n",
       "        -3.54118488e-05, -2.60254932e-03, -1.05338358e-03, -1.40510184e-03,\n",
       "         3.12488115e-04,  1.05983277e-03, -1.17258421e-03,  3.40598003e-04,\n",
       "         3.47204643e-04, -1.01909551e-03, -1.49627506e-03, -5.99342169e-04,\n",
       "         6.21019890e-05,  2.91233697e-03,  3.47954571e-04, -1.11085722e-03,\n",
       "        -1.11339909e-03, -1.22793786e-03, -1.65267230e-05,  2.01981663e-03,\n",
       "        -6.35857090e-05,  1.99642278e-03, -2.41719618e-04, -3.63284282e-04,\n",
       "         3.43131709e-04, -2.06543748e-03, -1.17323447e-03, -7.81952126e-04,\n",
       "         2.34112642e-03,  1.24925232e-03, -1.67312642e-04,  8.09558048e-04,\n",
       "         1.07922736e-03, -9.22502938e-04, -7.55231506e-04,  6.42494759e-04,\n",
       "        -3.18013207e-04,  1.61568812e-03,  1.96833402e-03,  2.04434867e-03,\n",
       "        -6.77746271e-04, -4.01314042e-03, -3.07845867e-03, -6.19861036e-04,\n",
       "         1.77812690e-03,  5.40185083e-04, -1.36247770e-03,  1.03022917e-05,\n",
       "         1.74605171e-04,  5.76850976e-04,  1.20365169e-03,  1.17877942e-03,\n",
       "        -5.41843824e-04,  1.04429827e-03,  4.44631833e-05,  1.09621385e-04,\n",
       "        -8.19094949e-04,  3.78100861e-04,  1.48592703e-03,  1.84525164e-03,\n",
       "        -1.51249381e-05, -1.41953371e-03, -1.48526321e-03, -2.68609001e-04,\n",
       "         9.38988636e-04,  6.79816212e-04,  1.97003885e-03, -1.55129933e-03,\n",
       "         8.02575378e-04,  1.73278900e-03, -6.10889511e-04, -8.01017597e-04,\n",
       "         3.32965356e-04,  1.04000039e-03, -1.45070984e-04,  6.08022473e-04,\n",
       "        -1.05794837e-03,  2.38657439e-03, -2.86729324e-03, -2.24356777e-03,\n",
       "        -3.79644503e-04,  9.35254804e-04,  1.89716910e-03, -2.04399707e-03,\n",
       "         8.10820856e-04,  1.39444474e-03, -1.04710725e-03, -6.81833034e-04,\n",
       "        -1.57258027e-03, -2.29071878e-03,  7.00365091e-04,  1.72781001e-03,\n",
       "        -7.50338559e-04, -9.20364189e-04,  2.21774707e-03, -1.78547584e-03,\n",
       "         6.41424027e-04,  9.34823986e-04,  4.47289071e-04,  1.57061548e-03,\n",
       "        -4.90281002e-05,  2.01786918e-03, -1.09254743e-03,  6.10100102e-04,\n",
       "         1.56734960e-03, -3.46534615e-03, -3.38424160e-04,  6.91410784e-04,\n",
       "        -8.38910257e-04,  9.39793104e-05, -6.86052559e-04, -4.19478281e-04,\n",
       "        -7.49401534e-04,  6.80066092e-04,  6.24572133e-04,  2.71185922e-04,\n",
       "        -1.20601128e-03,  2.57578810e-03, -1.03080526e-03, -3.15458401e-04,\n",
       "        -1.45108706e-03,  3.22792742e-04, -3.15982095e-04,  2.09742117e-03,\n",
       "         5.79169759e-05, -9.68753241e-04, -3.55882991e-04, -9.60669809e-04,\n",
       "        -8.23006062e-04,  2.08977211e-04,  9.15072778e-04,  1.03827445e-04,\n",
       "         1.08550511e-03, -1.73868239e-04, -2.94153030e-03, -1.02703365e-03,\n",
       "        -6.79819589e-04, -8.88537475e-04, -5.55094635e-04,  2.10451960e-03,\n",
       "         1.10716915e-03,  4.41458912e-04,  4.49678483e-04,  1.73230648e-03,\n",
       "        -1.05444030e-03, -9.32198258e-04,  9.83353700e-04, -1.66399500e-03,\n",
       "         1.16869943e-03,  1.17585813e-04,  2.24535619e-03,  6.41214174e-04,\n",
       "        -1.15591531e-03, -1.03430854e-03, -3.28148531e-05,  1.58907684e-03,\n",
       "        -1.29890882e-03, -1.26900613e-03,  2.85589323e-03,  1.58262794e-03,\n",
       "         7.27071678e-04, -1.46868083e-03,  2.67136946e-03, -6.67262558e-05,\n",
       "        -7.06047348e-04, -2.69430313e-04, -1.40938383e-03,  9.50657365e-05,\n",
       "         1.32722304e-04,  1.21181822e-03,  1.20089957e-03, -2.43323164e-03,\n",
       "        -9.12795606e-04, -2.11386548e-03,  6.64316527e-04, -1.24093045e-03,\n",
       "        -1.86551002e-03, -3.18324856e-03,  1.21288374e-03, -2.45284539e-03,\n",
       "        -1.90409887e-03,  1.13448479e-03,  1.38056790e-03,  2.52069702e-04,\n",
       "        -8.72750863e-05, -1.38392349e-04,  2.16907036e-03, -8.85922609e-04,\n",
       "        -1.21799590e-03,  4.56912108e-04, -2.01655955e-03,  3.54452083e-04,\n",
       "         1.27159825e-03,  5.59644225e-04, -5.26908886e-05,  1.26081172e-03,\n",
       "        -2.04340616e-03, -2.14160203e-03, -8.95370827e-04, -1.13985117e-03,\n",
       "         5.78936768e-04, -1.57455766e-03, -7.88734135e-04,  1.54324061e-03,\n",
       "        -2.69717336e-04,  7.08091339e-04, -4.14610328e-04,  8.53445900e-04,\n",
       "         1.23110085e-03,  1.33333177e-03,  2.53937868e-05,  3.71300248e-04,\n",
       "        -1.30248316e-03,  3.40455649e-04,  2.29809622e-03, -1.89873378e-03,\n",
       "         9.79177375e-04, -9.30011784e-04,  1.48562256e-03, -2.96604377e-04,\n",
       "         1.99188657e-03, -8.75696399e-04,  1.06612891e-03,  2.21806636e-03,\n",
       "        -9.00297235e-04, -2.09548149e-03,  1.24406611e-03,  4.11863260e-04,\n",
       "         8.25301807e-04, -1.31835223e-03, -1.02230561e-03, -1.64552139e-03,\n",
       "        -1.18075798e-03, -2.17716749e-03, -5.37482897e-04, -2.76935416e-04,\n",
       "        -1.41353522e-03, -3.92916730e-04,  1.33591660e-03,  1.47602515e-03,\n",
       "        -2.09989975e-03,  4.08917758e-04,  4.54698200e-04,  1.90012287e-03,\n",
       "        -5.21393002e-04, -9.82018499e-04, -1.40866105e-03,  7.43127267e-04,\n",
       "        -2.19155506e-03, -1.35334177e-04, -6.67144794e-05, -8.86259602e-05,\n",
       "         1.38557012e-03,  6.94097449e-04,  1.12207666e-03,  1.16577432e-03])}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_batch(0, init_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.2306458574418144e-08\n",
      "-1.6355168632603556e-08\n",
      "-8.216656849704962e-09\n",
      "-1.5105872691600908e-08\n",
      "-1.3843845863073523e-08\n",
      "-1.5823037522594285e-08\n",
      "-5.587758712330415e-09\n",
      "-9.044758398843444e-09\n",
      "-2.5928662732502272e-09\n",
      "-1.5792532510883195e-08\n"
     ]
    }
   ],
   "source": [
    "# b1验证\n",
    "for i in range(10):\n",
    "    img_i = np.random.randint(train_num)\n",
    "    test_parameters = init_parameters()\n",
    "    derivative = grad_parameters(train_img[img_i], train_lab[img_i], test_parameters)['b1']\n",
    "    value1 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)\n",
    "    test_parameters[1]['b'][i] += h\n",
    "    value2 = sqr_loss(train_img[img_i], train_lab[img_i], test_parameters)\n",
    "    print(derivative[i] - (value2 - value1) / h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 开始正式训练，先初始化所有参数\n",
    "parameters = init_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "# 将学习率*梯度作用在每个参数上，实现梯度下降\n",
    "def combine_parameters(parameters, grad, learn_rate):\n",
    "    # 拷贝参数，然后梯度下降，然后返回\n",
    "    parameter_tmp = copy.deepcopy(parameters)\n",
    "    parameter_tmp[0]['b'] -= learn_rate * grad['b0']\n",
    "    parameter_tmp[1]['b'] -= learn_rate * grad['b1']\n",
    "    parameter_tmp[1]['w'] -= learn_rate * grad['w1']\n",
    "    return parameter_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'b': array([-1.71834235e-03,  7.68177692e-05,  4.18779634e-04, -1.75905609e-03,\n",
       "         -9.94883453e-04, -5.75943916e-04, -6.31685969e-04,  4.91372607e-04,\n",
       "         -9.18192712e-04, -1.16583261e-03, -6.69873312e-05,  1.16782832e-03,\n",
       "         -2.92742555e-04, -1.72514581e-04,  1.11232176e-03,  1.85224060e-04,\n",
       "         -2.39017124e-04,  9.39578104e-04,  2.94182673e-04, -6.70110530e-04,\n",
       "          6.26790178e-04, -2.96297207e-03, -1.97536759e-03, -1.70710360e-03,\n",
       "          6.01255431e-05,  2.47853176e-03,  1.08226673e-03,  1.50397474e-04,\n",
       "          1.67987137e-03,  1.06462333e-03, -1.52141867e-03,  6.00880908e-05,\n",
       "          1.02752681e-03, -2.74662530e-04, -2.37116142e-03, -8.99287075e-04,\n",
       "         -2.41497172e-04, -2.56549171e-04,  2.58686473e-04,  4.01271993e-04,\n",
       "         -4.22826022e-04,  4.36625673e-04,  1.01428934e-03, -3.92645812e-04,\n",
       "         -1.29262753e-03,  2.55417808e-03,  1.52601314e-03,  1.67565681e-03,\n",
       "         -2.91371800e-03,  6.21485950e-04,  1.29648662e-03, -7.48116369e-05,\n",
       "         -8.37454027e-04, -1.26706279e-04, -6.32160130e-04,  3.43079928e-03,\n",
       "          3.59992718e-04,  2.15955473e-04,  1.64049842e-03, -6.44066231e-04,\n",
       "         -8.25451341e-04, -9.09396401e-05, -1.04433119e-03,  1.20710741e-03,\n",
       "          6.27025602e-04, -4.14916096e-04,  6.38764659e-04,  8.72512744e-04,\n",
       "          1.58111212e-03,  3.14745919e-05,  1.42254499e-03,  1.10699747e-03,\n",
       "          1.48271296e-03, -1.09578264e-03,  1.02791265e-03,  1.81811139e-03,\n",
       "          1.36663733e-03, -1.82426333e-03, -6.81145988e-04,  8.17607783e-04,\n",
       "         -1.10893023e-03,  1.45031118e-03,  1.78030099e-03,  3.33138712e-04,\n",
       "         -8.90956338e-05,  2.10528871e-03, -1.57843025e-03,  1.52049125e-03,\n",
       "          1.32793226e-03,  1.55545712e-04,  9.62199547e-04,  1.01284346e-04,\n",
       "          1.71766662e-04, -1.79450430e-03, -1.47751592e-03, -2.87295103e-04,\n",
       "          6.50811050e-04,  7.74322791e-04, -2.22962979e-04, -8.38219739e-04,\n",
       "          6.16093262e-04, -9.79692122e-04, -3.85959974e-04,  3.17380467e-04,\n",
       "         -1.48832338e-03,  3.26617261e-04,  8.84651463e-04,  5.56055046e-04,\n",
       "         -1.66345730e-04,  2.07150538e-03,  6.34562239e-04,  9.10968505e-04,\n",
       "         -1.70967979e-03, -5.81155080e-04, -1.63055537e-03, -4.06924226e-04,\n",
       "          2.31853652e-03,  1.98259788e-04,  8.88457749e-04, -2.11944132e-04,\n",
       "          1.25515999e-04,  8.18983703e-04,  5.84747440e-04, -3.96247071e-04,\n",
       "          1.40994160e-03, -5.44295019e-04, -9.70728345e-04,  4.13155780e-04,\n",
       "         -3.20134164e-04, -3.54201204e-04, -8.04713603e-04, -4.01550072e-04,\n",
       "          5.41155466e-04, -9.44294142e-04, -1.22557998e-03,  1.86498680e-04,\n",
       "         -1.38181247e-03,  1.63879559e-03,  5.40524742e-04, -2.85351524e-03,\n",
       "         -4.66059671e-04, -1.10076415e-03, -1.73108933e-03,  1.71938056e-04,\n",
       "         -1.40179219e-03,  1.19034497e-03,  3.71797819e-04,  1.04481608e-03,\n",
       "         -1.10759303e-03, -9.19520925e-04, -5.65039555e-05,  1.12001598e-03,\n",
       "          1.64807314e-03,  1.39095117e-03, -1.48742471e-03,  9.33918692e-04,\n",
       "          3.33583262e-04, -1.21124589e-03, -8.71558803e-04,  9.40323885e-04,\n",
       "         -3.59286660e-04,  4.81955769e-04, -2.41557075e-03, -7.73008679e-04,\n",
       "         -1.16893719e-03,  3.55645148e-05,  6.42735221e-05, -1.48151470e-03,\n",
       "          9.84325704e-04, -9.32440118e-04,  9.79049655e-04, -5.70928487e-04,\n",
       "         -5.47170842e-04,  1.58487069e-03, -3.78203267e-04, -1.43522076e-03,\n",
       "          8.14697055e-04,  1.07724549e-03,  9.04820311e-04,  1.47601454e-03,\n",
       "         -1.75248219e-03,  7.88494044e-06, -1.43057096e-03, -1.34685011e-03,\n",
       "         -8.47243853e-04, -2.34378099e-04, -9.33809611e-04, -1.06219278e-03,\n",
       "         -1.98434800e-03,  2.69261774e-03, -2.76586747e-03, -1.47279812e-03,\n",
       "         -9.55995204e-04,  1.36467453e-03, -2.30140352e-03, -4.69280896e-04,\n",
       "          6.32826722e-05, -4.01952540e-04,  2.54156575e-03, -1.70114968e-04,\n",
       "          5.64468674e-04, -1.79704309e-04, -7.92787780e-04, -5.47868571e-04,\n",
       "         -8.46135809e-04,  1.86311624e-03,  8.52857767e-04,  4.46107255e-04,\n",
       "         -9.77405251e-04,  1.03630489e-03, -6.36250089e-04,  5.79541935e-04,\n",
       "         -9.29863767e-05, -1.03011795e-03,  3.93036781e-04,  5.34714953e-04,\n",
       "          3.02673069e-04, -1.04809718e-03,  1.53701032e-04, -1.81219685e-03,\n",
       "         -1.16993121e-04,  1.26555607e-03, -1.32761560e-03, -1.30430657e-03,\n",
       "          2.51436693e-04,  1.01383913e-03,  1.31616312e-03,  2.66975363e-04,\n",
       "         -3.95007012e-04,  3.66989953e-04,  5.45058153e-05,  4.32714154e-04,\n",
       "         -1.29161192e-03,  1.74230136e-03,  1.11057180e-03,  4.72403503e-04,\n",
       "         -1.17317814e-04, -1.59883832e-04, -2.92989037e-05, -3.73859494e-04,\n",
       "         -8.68289815e-04,  8.44082079e-04,  9.18812706e-04, -1.52319381e-04,\n",
       "         -1.64637132e-04, -1.05890649e-05,  5.99422495e-04,  3.54242373e-04,\n",
       "         -1.26945867e-03, -6.77878210e-04, -2.64178154e-03, -9.48422952e-04,\n",
       "          1.67238662e-05,  2.00141500e-03,  1.66860665e-03, -5.19966531e-04,\n",
       "         -8.25046904e-04, -6.86315303e-04, -1.54250166e-03,  1.03651967e-03,\n",
       "         -1.66844843e-03,  1.97305408e-04, -6.85458510e-04,  1.73459548e-04,\n",
       "          1.37278110e-04, -1.46225546e-03,  5.77901641e-04,  8.98898164e-04,\n",
       "         -2.00801842e-04,  4.83080259e-04,  4.14589037e-04,  8.67245298e-05,\n",
       "         -1.76887375e-04,  8.71361017e-04, -3.77688874e-04,  2.95407966e-03,\n",
       "         -1.33848100e-03, -1.56221765e-04,  4.57703987e-04, -4.96008954e-04,\n",
       "         -9.73240614e-04, -1.23337150e-03,  7.50332614e-04,  4.22344438e-04,\n",
       "          1.80723657e-03,  6.85126233e-04, -1.06786202e-04,  1.36089435e-03,\n",
       "         -2.65932341e-04, -3.86100144e-04, -9.98255578e-04, -4.59485672e-04,\n",
       "          1.85092547e-04,  4.83892612e-04, -6.42222306e-04, -7.59261751e-05,\n",
       "          3.90207645e-04,  9.09128253e-05,  1.00809375e-04, -1.30765201e-03,\n",
       "         -1.25029550e-03, -1.12910972e-03,  1.97527272e-03,  3.44897922e-04,\n",
       "          9.31052176e-04, -1.59173910e-03, -2.15302533e-03, -7.46932573e-04,\n",
       "          1.52000106e-03,  5.44873311e-04, -1.40401210e-03, -4.46520476e-04,\n",
       "         -7.04670555e-04, -3.14794575e-04, -9.68710712e-04,  3.76779771e-04,\n",
       "          1.58735065e-04,  4.93626917e-04,  1.20467195e-03,  4.27860244e-05,\n",
       "          1.12905559e-04, -1.10509714e-03, -2.35276170e-03,  1.99739433e-03,\n",
       "         -2.16945946e-05, -2.63237956e-04, -1.32016863e-03,  3.26235190e-04,\n",
       "         -2.99063164e-04, -5.10476200e-04,  1.37783114e-03, -2.29668873e-03,\n",
       "         -1.56704394e-03,  8.21937545e-04, -1.37633855e-03, -1.83765108e-03,\n",
       "         -8.28562076e-04, -1.61478361e-03, -2.21945292e-04, -1.94096787e-04,\n",
       "          2.18888406e-03, -4.93705646e-04,  1.10342141e-03,  2.85493300e-04,\n",
       "          1.06601102e-04,  4.31135721e-04,  6.17317595e-04,  2.61574303e-04,\n",
       "          6.25604418e-05,  1.03952000e-03,  2.22057982e-03, -1.60821033e-03,\n",
       "          1.64743223e-03,  8.66460323e-04, -2.83089484e-04,  3.80837620e-04,\n",
       "         -1.37156029e-03,  1.04543379e-03, -2.43656310e-04,  8.26710496e-04,\n",
       "          3.03570760e-04,  8.83466146e-04,  1.76909259e-04,  1.60931306e-03,\n",
       "         -1.57935335e-03, -3.78693016e-04, -3.65683558e-04,  1.25130279e-04,\n",
       "         -1.12399158e-03, -8.85353552e-05,  2.30360082e-04,  4.01261579e-04,\n",
       "          5.19115067e-04,  7.15486346e-04, -2.15911698e-05, -1.07478182e-03,\n",
       "         -7.02719446e-05,  2.99083968e-03,  1.12260783e-03,  2.66477834e-05,\n",
       "         -6.12561862e-04,  5.05712365e-04, -8.79766717e-04, -1.78435236e-04,\n",
       "          8.69483869e-04, -1.30456027e-03,  2.98886574e-05,  2.79799108e-05,\n",
       "         -2.71249801e-04, -1.26538246e-05,  6.02418895e-04,  2.60954532e-03,\n",
       "         -6.13635297e-04, -1.84083836e-04,  8.57839431e-04, -9.78958583e-04,\n",
       "          1.61317696e-03, -5.00354624e-05, -1.11539707e-03,  1.04233514e-03,\n",
       "          1.10874111e-03, -4.16558964e-04,  2.23991106e-03,  9.27082893e-05,\n",
       "         -7.66284146e-04, -7.46757185e-04, -3.58344696e-04,  2.36407293e-04,\n",
       "          1.39476806e-03,  2.09809082e-04, -8.28403419e-05, -5.91444675e-04,\n",
       "          2.66515710e-04, -2.65626396e-03, -1.06736326e-04, -6.24007160e-04,\n",
       "         -7.42479808e-04, -1.35241838e-03,  1.46679108e-03, -1.38593850e-04,\n",
       "         -1.05171099e-03,  3.21424271e-04, -9.08623542e-04,  2.08619292e-03,\n",
       "         -8.48736923e-04,  4.93732059e-04, -2.79472946e-04, -1.55957967e-03,\n",
       "          1.00667624e-03,  1.14016448e-03,  4.66390968e-04, -2.14624497e-04,\n",
       "         -2.53290006e-03, -2.23621699e-03,  5.95184947e-04, -5.64393502e-04,\n",
       "          1.31408815e-03, -2.66187223e-03, -7.55777988e-04, -4.12549077e-04,\n",
       "          2.61015699e-04, -4.54607396e-05,  8.95222015e-04, -9.51092389e-05,\n",
       "          1.01298179e-03, -1.46072327e-03,  6.23207885e-04,  2.18666545e-04,\n",
       "          1.04375291e-03,  1.82477340e-03, -6.36668621e-04,  7.01071526e-04,\n",
       "          4.09755534e-04,  2.37961541e-04, -1.30344514e-03, -5.87960792e-04,\n",
       "          8.38459286e-04, -7.22688657e-04, -3.91573509e-04,  8.02135294e-04,\n",
       "          2.39772445e-03, -2.71427147e-04, -1.27557951e-03, -3.11128356e-04,\n",
       "         -9.15411066e-06, -2.32087458e-03,  1.25972845e-03, -8.71341377e-04,\n",
       "          1.68134390e-04, -3.26696792e-05,  1.71896944e-03,  8.67514817e-04,\n",
       "          1.84364904e-03, -1.47636368e-03,  1.23582045e-03, -1.73576277e-03,\n",
       "         -4.05174216e-04,  3.06168816e-04, -2.70820328e-04, -1.19539178e-03,\n",
       "          8.61818313e-04, -4.81447446e-04, -6.36857371e-04,  4.85500391e-04,\n",
       "          7.46825122e-04, -9.20638800e-04, -2.12869961e-03,  2.17910252e-03,\n",
       "         -6.22124231e-04,  1.32331475e-03,  1.22010712e-03,  1.44996915e-04,\n",
       "         -3.29795670e-05, -3.53361651e-05, -1.20729234e-03, -6.38309779e-04,\n",
       "         -1.26585081e-03, -5.32942291e-05, -1.43235308e-03,  2.95886640e-04,\n",
       "         -1.37751579e-03,  1.50422805e-03,  8.84225762e-04,  1.74805888e-04,\n",
       "          1.22676423e-03, -1.74409829e-03, -3.22981875e-04, -1.09236568e-03,\n",
       "          9.19613343e-04,  4.92610583e-05, -2.35779857e-04,  1.48101633e-03,\n",
       "          1.69725431e-04,  1.07223841e-03,  8.25037747e-05, -1.21697317e-03,\n",
       "         -3.49749318e-04, -8.69451291e-04, -1.12318918e-03, -1.25113189e-03,\n",
       "          2.02460008e-04, -1.35787493e-03,  1.43671730e-03,  8.33469939e-04,\n",
       "         -8.62384157e-04,  6.00384944e-04,  7.91452433e-04,  1.58622964e-03,\n",
       "          1.27364115e-03, -1.06105216e-03, -9.34618708e-04,  1.69167215e-03,\n",
       "          1.43253094e-03,  1.35078515e-05, -2.02724385e-03, -6.11289652e-04,\n",
       "         -7.74113097e-04, -1.42935914e-04, -6.27811650e-04,  9.71175925e-04,\n",
       "         -7.95359229e-04,  7.31730184e-04,  2.42067200e-03, -1.38362229e-05,\n",
       "          1.25604271e-03,  1.08033982e-03, -1.11549755e-03,  2.30565729e-04,\n",
       "         -1.07143860e-03,  4.81737959e-04, -2.14080465e-05,  1.12066975e-03,\n",
       "          2.61428150e-04, -1.18823789e-03, -6.29452673e-05,  1.16832839e-03,\n",
       "          1.27224796e-03, -4.06774301e-05, -1.09266708e-03, -4.00325500e-04,\n",
       "          1.43453660e-03, -5.87935660e-04, -1.59623082e-03, -2.41836425e-04,\n",
       "          5.86781783e-05, -1.07543981e-03,  1.39604669e-03, -5.00614949e-04,\n",
       "         -4.13785963e-04, -4.28529461e-05, -2.44017982e-04, -1.77132284e-03,\n",
       "          9.85193938e-04, -8.31654783e-04,  3.91912336e-05,  8.12812096e-04,\n",
       "          1.16145413e-03,  2.92872244e-04, -1.81943989e-03, -5.90051149e-06,\n",
       "          4.04394257e-04, -1.01086979e-04,  8.02752241e-04, -9.26399620e-04,\n",
       "         -8.22651695e-05, -2.81195641e-04, -1.34589424e-03, -4.13268112e-04,\n",
       "          2.27367912e-03, -8.94226762e-04,  2.66413777e-04, -1.19755489e-03,\n",
       "          1.84860420e-03, -3.48904146e-04,  1.09656134e-03, -9.65961645e-04,\n",
       "         -4.87791098e-05, -9.78636966e-05,  6.72192769e-04, -1.95472375e-03,\n",
       "          4.45861344e-04,  9.24896748e-04,  9.93653746e-04, -3.43123101e-05,\n",
       "          8.70846133e-04,  7.45924001e-04,  4.05950470e-05, -4.64695529e-04,\n",
       "         -1.26678086e-03,  5.09671908e-04, -2.11523210e-03,  1.29129512e-03,\n",
       "         -1.94223669e-03, -1.73378245e-04, -2.49412707e-04,  2.29448119e-04,\n",
       "          2.04568281e-03,  9.28746268e-04, -6.67608794e-05, -2.72344941e-04,\n",
       "         -3.39270405e-04,  4.12327308e-04,  3.07418935e-04, -6.73941665e-04,\n",
       "          6.83515657e-04,  9.48467971e-04,  3.79560542e-04,  3.59293644e-04,\n",
       "          2.64602622e-04, -4.36353275e-04, -8.28950899e-05, -7.39541368e-04,\n",
       "          1.13829468e-03,  1.79667121e-04, -9.76409560e-04, -1.36468587e-04,\n",
       "          1.22234347e-03,  1.20808333e-03,  1.48581513e-03,  1.68639783e-04,\n",
       "          7.25313517e-04,  3.23305985e-04, -9.34104919e-04,  8.66584987e-04,\n",
       "          1.51945415e-03, -6.28556754e-04,  1.92369124e-03, -1.54424316e-03,\n",
       "          6.10574933e-04, -5.40943741e-04, -1.50145756e-03,  1.73215298e-04,\n",
       "         -1.03460183e-03,  3.53474962e-04,  9.20921679e-05,  1.33909957e-04,\n",
       "         -6.47204300e-04, -2.03174233e-04,  2.23777668e-04,  1.83856616e-03,\n",
       "          2.17617424e-03, -3.36698113e-04,  1.39809272e-03,  1.06100342e-03,\n",
       "         -9.20784198e-05,  3.79596253e-04, -1.35472342e-04, -6.51925183e-04,\n",
       "         -1.97699470e-03,  1.01424756e-03, -1.15218977e-03, -7.78888493e-04,\n",
       "          3.63489641e-04, -1.03214449e-03, -1.04920804e-03, -1.41977806e-03,\n",
       "         -5.68698224e-04, -7.17683718e-04, -5.93208465e-04, -1.74558518e-03,\n",
       "          1.30262771e-04,  4.62775316e-04,  4.70683339e-04,  1.92561166e-03,\n",
       "         -1.01224522e-03,  2.23564257e-03,  1.85244394e-03, -1.38698684e-03,\n",
       "         -1.61537205e-03,  1.79520092e-04,  2.21755149e-03,  3.89426207e-04,\n",
       "         -7.28266350e-04,  9.08941475e-04,  6.50256589e-04, -5.47845254e-04,\n",
       "          1.32189073e-03,  9.70240028e-04, -8.95895520e-04, -5.81535803e-04,\n",
       "         -1.17177680e-03,  6.51062862e-04,  4.29517024e-04, -1.16156912e-03,\n",
       "          1.71188420e-03, -1.32019715e-03, -7.91916881e-04,  8.23591835e-04,\n",
       "         -3.99873633e-04,  7.06762324e-04,  4.80419565e-04,  3.55236630e-04,\n",
       "          1.24237958e-03,  6.67756539e-05, -1.34950423e-03, -1.79522976e-03,\n",
       "         -7.23842059e-04, -4.72657360e-04, -4.42462609e-04, -8.20639323e-04,\n",
       "         -2.05379633e-04,  4.20716069e-05,  2.16777185e-04, -1.29729518e-03,\n",
       "         -9.71806220e-04, -2.09577622e-05, -1.71889191e-03,  1.72338877e-03,\n",
       "         -1.67033762e-03, -2.66474546e-04, -3.30137603e-04, -6.67091460e-05,\n",
       "         -4.58334808e-04,  2.09120049e-03,  1.16732389e-03, -5.27903361e-04,\n",
       "          4.07368154e-04, -1.32931577e-04,  1.22731090e-03, -2.78612510e-03,\n",
       "         -4.71744484e-04, -2.04113659e-03, -1.50868440e-03, -8.86167966e-04,\n",
       "          1.22004428e-03,  8.06690315e-04,  2.10009144e-03,  1.21518245e-03,\n",
       "          5.21592988e-04,  1.15715132e-03,  8.23088398e-04, -3.60593187e-04,\n",
       "          2.10566143e-03,  1.39174989e-03,  2.78702807e-04,  1.44842933e-04,\n",
       "          1.43499249e-03, -1.22358183e-03, -1.65760992e-03, -1.37761685e-03,\n",
       "          5.93679946e-04, -7.10968113e-04, -1.92847933e-05,  4.01518377e-04,\n",
       "          7.41558201e-04, -3.15768342e-04,  1.49753105e-03,  3.56250174e-05,\n",
       "         -1.99245274e-03,  3.04189729e-03,  6.32174345e-04,  2.06874577e-03,\n",
       "          1.89781097e-03, -5.00143939e-04,  8.55543030e-04,  6.01785628e-04,\n",
       "         -4.69205112e-04,  1.07055104e-03, -8.54268206e-04, -9.05682549e-04,\n",
       "         -1.76422834e-04, -4.67119800e-04,  4.63919840e-04, -1.05073495e-03])},\n",
       " {'b': array([ 0.00848295, -0.00442194, -0.01479875,  0.00405839, -0.00621937,\n",
       "         -0.00285182,  0.00617522, -0.00179854,  0.00071714,  0.01065673]),\n",
       "  'w': array([[-0.08597068, -0.02055546,  0.08492502, ..., -0.00517476,\n",
       "           0.08245606,  0.06177858],\n",
       "         [-0.05756908, -0.08248979, -0.0750809 , ...,  0.05493108,\n",
       "           0.03702635, -0.03381442],\n",
       "         [ 0.01159525,  0.00954124,  0.02462906, ...,  0.03755413,\n",
       "           0.00759054,  0.07621193],\n",
       "         ...,\n",
       "         [-0.03258364,  0.07524495, -0.02007656, ..., -0.00223532,\n",
       "           0.06427671, -0.05496881],\n",
       "         [-0.05444965,  0.06681722,  0.00117286, ..., -0.07698962,\n",
       "          -0.0274646 ,  0.06765333],\n",
       "         [ 0.03547066,  0.07588154,  0.03326274, ...,  0.01185462,\n",
       "          -0.01347288, -0.06891906]])}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_parameters(parameters, train_batch(0, parameters), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = init_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation accuracy: 0.0722\n"
     ]
    }
   ],
   "source": [
    "valid_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running batch: 0 / 500\n",
      "running batch: 100 / 500\n",
      "running batch: 200 / 500\n",
      "running batch: 300 / 500\n",
      "running batch: 400 / 500\n",
      "done.\n",
      "validation accuracy: 0.9167\n"
     ]
    }
   ],
   "source": [
    "# 设置学习率为1\n",
    "learn_rate = 1\n",
    "# 开始训练\n",
    "for i in range(train_num // batch_size):\n",
    "    if not i % 100:\n",
    "        print(f'running batch: {i} / {train_num // batch_size}')\n",
    "    # 求每个参数的梯度\n",
    "    grad_tmp = train_batch(i, parameters)\n",
    "    # 将梯度和学习率作用在参数上实现梯度下降\n",
    "    parameters = combine_parameters(parameters, grad_tmp, learn_rate)\n",
    "print('done.')\n",
    "valid_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predict result: 5\n",
      "label: 5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAN9UlEQVR4nO3df6hc9ZnH8c9HE/9JqiQriUFl0xTBLYp2CbJgWbqWNm5AYxGXBrIoG0z/iNjKBlZdpIH1R5RNlxVRuDXSdKkpEaOGUGklmM36T8mNRM0PU5PgmltjYojYBMVo8uwf92S5jXe+c505M2dyn/cLLnPnPHPmPJzkc8+Z+c6ZryNCACa/85puAEB/EHYgCcIOJEHYgSQIO5DElH5uzDZv/QM9FhEeb3lXR3bbN9rea3uf7Xu7eS4AveVOx9ltny/pD5K+J2lE0jZJiyNid2EdjuxAj/XiyH6dpH0RcSAiTkr6taRFXTwfgB7qJuyXSjo45v5ItezP2F5me9j2cBfbAtClbt6gG+9U4Uun6RExJGlI4jQeaFI3R/YRSZePuX+ZpPe7awdAr3QT9m2SrrD9ddsXSPqhpI31tAWgbh2fxkfEF7bvkvRbSedLeiYidtXWGYBadTz01tHGeM0O9FxPPlQD4NxB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBIdT9mM+tx3333F+p133lmsL126tGVt6tSpxXVnzZpVrC9atKhYv/XWW4v1JtnjTmYqSVq9enVx3RUrVtTdTuO6CrvtdyUdl3RK0hcRMb+OpgDUr44j+99FxNEangdAD/GaHUii27CHpN/Z3m572XgPsL3M9rDt4S63BaAL3Z7GXx8R79ueJekV229HxNaxD4iIIUlDkmQ7utwegA51dWSPiPer2yOSXpB0XR1NAahfx2G3Pc321878Lun7knbW1RiAejmiszNr2/M0ejSXRl8OPBsRD7VZh9P4cbQbZ3/ooeJuRQdOnTpVrF955ZXF+v79++tsp1YRMe4HDDp+zR4RByRd03FHAPqKoTcgCcIOJEHYgSQIO5AEYQeS4BLXSe6zzz4r1p977rmunr/dJbSXXHJJy9oLL7zQsiZJu3bt6qinMy688MKWtQsuuKC47iAPrXWKIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJNHxJa4dbYxLXMe1cePGYn327NnF+tNPP92y1m4s++jR7r4rtPR1ze3qp0+f7mrbGF+rS1w5sgNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoyz90G7a6d3795drG/btq1YX7x48VfuCZMX4+xAcoQdSIKwA0kQdiAJwg4kQdiBJAg7kATfG98H8+fPL9bnzZtXrK9fv77OdpBU2yO77WdsH7G9c8yymbZfsf1OdTujt20C6NZETuN/IenGs5bdK2lzRFwhaXN1H8AAaxv2iNgq6dhZixdJWlv9vlbSLTX3BaBmnb5mnx0RhyQpIg7ZntXqgbaXSVrW4XYA1KTnb9BFxJCkISnvhTDAIOh06O2w7TmSVN0eqa8lAL3Qadg3Srq9+v12SS/V0w6AXml7Gm97naTvSLrY9oikn0paJWm97aWS3pN0Wy+bPNctWbKkq/W3bt1aUyfIrG3YI6LVNyN8t+ZeAPQQH5cFkiDsQBKEHUiCsANJEHYgCS5x7YNp06YV6x9++GGx/tprrxXr06dPb1m76KKLius2+TXU7b4ie/v27cX6iRMn6mxn0uPIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMGVzH5w8ebJYP378eLH+8MMPF+vLly9vWZs7d25x3UG2b9++Yv2GG24o1kdGRups55zBlM1AcoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7DWYOXNmsf7BBx8U61OmnLtfK/D2228X659++mnL2tVXX11ct91+2bJlS7G+YMGClrXPP/+8uO65jHF2IDnCDiRB2IEkCDuQBGEHkiDsQBKEHUji3B3gHSDz5s0r1s87r7d/Uw8ePNiytmHDhuK67733XrH+6quvFut79+4t1kvj7FdddVVx3QceeKBYv+228kzhjz32WMvaPffcU1x3Mmr7v9D2M7aP2N45ZtlK23+0vaP6WdjbNgF0ayKHnF9IunGc5f8REddWP7+pty0AdWsb9ojYKulYH3oB0EPdvJi8y/ab1Wn+jFYPsr3M9rDt4S62BaBLnYb9KUnfkHStpEOSVrd6YEQMRcT8iJjf4bYA1KCjsEfE4Yg4FRGnJf1c0nX1tgWgbh2F3facMXd/IGlnq8cCGAxtr2e3vU7SdyRdLOmwpJ9W96+VFJLelfSjiDjUdmOT9Hr2dvbv31+sz5kzp1gfGhoq1h9//PGWtQMHDhTXHWTtPr+wdu3aYr30nfnXXHNNcd1jx87d96RbXc/e9kM1EbF4nMVruu4IQF/xcVkgCcIOJEHYgSQIO5AEYQeS4Kuk+2D69OnF+tSpU4v1jz76qM52Jo0VK1YU66VLXJ944oniunfffXdHPQ0CvkoaSI6wA0kQdiAJwg4kQdiBJAg7kARhB5Lgq6T74MSJE023MCnt2rWr43XbXeI6GXFkB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGGfvg8suu6xYHxkZ6VMnOKPdv8lkxJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnL0PHnnkkWJ95cqVxXq7KZ8nqwULFhTrDz74YMfP/eijj3a87rmq7ZHd9uW2X7W9x/Yu2z+uls+0/Yrtd6rbGb1vF0CnJnIa/4Wkf46Iv5L0N5KW2/6mpHslbY6IKyRtru4DGFBtwx4RhyLi9er345L2SLpU0iJJa6uHrZV0S6+aBNC9r/Sa3fZcSd+S9HtJsyPikDT6B8H2rBbrLJO0rLs2AXRrwmG3PV3S85J+EhF/ssedO+5LImJI0lD1HCkndgQGwYSG3mxP1WjQfxURG6rFh23PqepzJB3pTYsA6tD2yO7RQ/gaSXsi4mdjShsl3S5pVXX7Uk86nAQ++eSTYn3Lli3F+po1a4r1N954o2XtxRdfLK7b7ZTdU6aU/wuVzgBXr15dXPeOO+4o1ttNhX3s2LGWtc2bNxfXnYwmchp/vaR/lPSW7R3Vsvs1GvL1tpdKek/Sbb1pEUAd2oY9Il6T1OrP83frbQdAr/BxWSAJwg4kQdiBJAg7kARhB5Jwt+OsX2ljST9Bd9NNNxXrzz77bLE+bdq0jre9adOmYv3jjz/u+LklaeHChcX6jBm9uxjy6NGjxXqpt+Hh4brbGRgRMe7oGUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYB8NRTTxXrS5YsKda7GYdv0uHDh4v1J598slh/+eWXi/XJPJZewjg7kBxhB5Ig7EAShB1IgrADSRB2IAnCDiTBOPs54Oabby7WV61a1bI2d+7c4rrr1q3rpKUJO3jwYMtau2vts46Td4txdiA5wg4kQdiBJAg7kARhB5Ig7EAShB1Iou04u+3LJf1S0iWSTksaioj/tL1S0p2SPqween9E/KbNczHODvRYq3H2iYR9jqQ5EfG67a9J2i7pFkn/IOlERPz7RJsg7EDvtQr7ROZnPyTpUPX7cdt7JF1ab3sAeu0rvWa3PVfStyT9vlp0l+03bT9je9x5fmwvsz1sm88+Ag2a8GfjbU+X9N+SHoqIDbZnSzoqKST9m0ZP9f+pzXNwGg/0WMev2SXJ9lRJmyT9NiJ+Nk59rqRNEXFVm+ch7ECPdXwhjG1LWiNpz9igV2/cnfEDSTu7bRJA70zk3fhvS/ofSW9pdOhNku6XtFjStRo9jX9X0o+qN/NKz8WRHeixrk7j60LYgd7jenYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASbb9wsmZHJf3vmPsXV8sG0aD2Nqh9SfTWqTp7+8tWhb5ez/6ljdvDETG/sQYKBrW3Qe1LordO9as3TuOBJAg7kETTYR9qePslg9rboPYl0Vun+tJbo6/ZAfRP00d2AH1C2IEkGgm77Rtt77W9z/a9TfTQiu13bb9le0fT89NVc+gdsb1zzLKZtl+x/U51O+4cew31ttL2H6t9t8P2woZ6u9z2q7b32N5l+8fV8kb3XaGvvuy3vr9mt32+pD9I+p6kEUnbJC2OiN19baQF2+9Kmh8RjX8Aw/bfSjoh6Zdnptay/ZikYxGxqvpDOSMi/mVAeluprziNd496azXN+B1qcN/VOf15J5o4sl8naV9EHIiIk5J+LWlRA30MvIjYKunYWYsXSVpb/b5Wo/9Z+q5FbwMhIg5FxOvV78clnZlmvNF9V+irL5oI+6WSDo65P6LBmu89JP3O9nbby5puZhyzz0yzVd3Oarifs7WdxrufzppmfGD2XSfTn3eribCPNzXNII3/XR8Rfy3p7yUtr05XMTFPSfqGRucAPCRpdZPNVNOMPy/pJxHxpyZ7GWucvvqy35oI+4iky8fcv0zS+w30Ma6IeL+6PSLpBY2+7Bgkh8/MoFvdHmm4n/8XEYcj4lREnJb0czW476ppxp+X9KuI2FAtbnzfjddXv/ZbE2HfJukK21+3fYGkH0ra2EAfX2J7WvXGiWxPk/R9Dd5U1Bsl3V79fruklxrs5c8MyjTeraYZV8P7rvHpzyOi7z+SFmr0Hfn9kv61iR5a9DVP0hvVz66me5O0TqOndZ9r9IxoqaS/kLRZ0jvV7cwB6u2/NDq195saDdachnr7tkZfGr4paUf1s7DpfVfoqy/7jY/LAknwCTogCcIOJEHYgSQIO5AEYQeSIOxAEoQdSOL/AEfaa1Xpfyl4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 验证训练结果\n",
    "selected_index = np.random.randint(valid_num)\n",
    "# 输入图片需要扁平化，输出是（类似）独热的，因为学习的是它向独热靠近的过程，所以取下标最大的就是结果\n",
    "predres = predict(valid_img[selected_index].flatten(), parameters).argmax()\n",
    "# 返回预测值\n",
    "print(f'predict result: {predres}')\n",
    "# 返回真实值和图片\n",
    "show_valid(selected_index)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
